Allow configuration for behavior when llms-full.txt is too large (#35)

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+0.5.0
+-----
+
+- Add :confval:`llms_txt_full_size_policy` configuration option to control behavior when :confval:`llms_txt_full_max_size` is exceeded.
+  `#35 <https://github.com/jdillard/sphinx-llms-txt/pull/35>`_
+
 0.4.1
 -----
 

docs/source/advanced-configuration.rst

@@ -76,15 +76,26 @@ Handling Large Documentation
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 For very large documentation sets, generating the full documentation file might exceed reasonable size limits.
-You can set a maximum line count:
+You can set a maximum line count and control what happens when that limit is exceeded:
 
 .. code-block:: python
 
    llms_txt_full_max_size = 10000  # Maximum 10,000 lines
+   llms_txt_full_size_policy = "warn_skip"  # Default behavior
 
-If the generated file would exceed this limit, the extension will skip its generation and show a warning, allowing the build to complete.
+The ``llms_txt_full_size_policy`` setting controls both the log level and action taken when the size limit is exceeded.
+It uses the format ``"<loglevel>_<action>"``:
 
-.. tip:: Use :ref:`excluding_content` to remove less relevant pages.
+**Log levels:**
+- ``warn``: Log as a warning (default)
+- ``info``: Log as informational message
+
+**Actions:**
+- ``skip``: Don't create the file (default)
+- ``keep``: Create the file anyway, ignoring the size limit
+- ``note``: Create a placeholder file explaining why the full file wasn't generated
+
+.. tip:: Use :ref:`excluding_content` to remove less relevant pages and reduce the file size.
 
 .. _custom_directive_handling:
 
@@ -198,6 +209,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
    llms_txt_filename = "ai-summary.txt"
    llms_txt_full_filename = "ai-full-docs.txt"
    llms_txt_full_max_size = 50000
+   llms_txt_full_size_policy = "warn_note"
 
    # Content customization
    llms_txt_title = "Project Documentation for AI Assistants"

docs/source/configuration-values.rst

@@ -24,11 +24,22 @@ Project Configuration Values
    - **Type**: integer or ``None``
    - **Default**: ``None`` (no limit)
    - **Description**: Sets a maximum line count for ``llms_txt_full_filename``.
-     If exceeded, the file is skipped and a warning is shown, but the build still completes.
+     Behavior when exceeded is controlled by :confval:`llms_txt_full_size_policy`.
      See :ref:`handling_large_documentation`.
 
    .. versionadded:: 0.2.0
 
+.. confval:: llms_txt_full_size_policy
+
+   - **Type**: string
+   - **Default**: ``'warn_skip'``
+   - **Description**: Controls what happens when :confval:`llms_txt_full_max_size` is exceeded.
+     Format is ``<loglevel>_<action>``. Log levels: ``warn``, ``info``.
+     Actions: ``skip``, ``keep``, ``note``.
+     See :ref:`handling_large_documentation`.
+
+   .. versionadded:: 0.5.0
+
 .. confval:: llms_txt_file
 
    - **Type**: boolean

sphinx_llms_txt/__init__.py

@@ -12,7 +12,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.4.1"
+__version__ = "0.5.0"
 
 # Export classes needed by tests
 __all__ = [
@@ -74,6 +74,7 @@ def build_finished(app: Sphinx, exception):
             "llms_txt_full_file": app.config.llms_txt_full_file,
             "llms_txt_full_filename": app.config.llms_txt_full_filename,
             "llms_txt_full_max_size": app.config.llms_txt_full_max_size,
+            "llms_txt_full_size_policy": app.config.llms_txt_full_size_policy,
             "llms_txt_directives": app.config.llms_txt_directives,
             "llms_txt_exclude": app.config.llms_txt_exclude,
             "llms_txt_code_files": app.config.llms_txt_code_files,
@@ -102,6 +103,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("llms_txt_full_file", True, "env")
     app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
     app.add_config_value("llms_txt_full_max_size", None, "env")
+    app.add_config_value("llms_txt_full_size_policy", "warn_skip", "env")
     app.add_config_value("llms_txt_directives", [], "env")
     app.add_config_value("llms_txt_title", None, "env")
     app.add_config_value("llms_txt_summary", None, "env")

sphinx_llms_txt/manager.py

@@ -273,16 +273,34 @@ class LLMSFullManager:
         added_files = set()
         total_line_count = code_files_line_count
         max_lines = self.config.get("llms_txt_full_max_size")
-        abort_due_to_max_lines = False
+
+        # Parse size_policy configuration early to determine collection strategy
+        size_policy_action = None
+        aborted_due_to_size = False
+        if max_lines is not None:
+            size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip")
+            _, size_policy_action = self._parse_size_policy_config(size_policy)
+
+        # Only collect all files if action is "keep"
+        # For "skip" and "note", we can abort early when size limit is exceeded
+        should_abort_early = size_policy_action in ["skip", "note"]
 
         for docname, _ in page_order:
             if docname in docname_to_file:
                 file_path = docname_to_file[docname]
                 content, line_count = self._read_source_file(file_path, docname)
 
-                # Check if adding this file would exceed the maximum line count
+                # Abort early for skip/note actions
-                if max_lines is not None and total_line_count + line_count > max_lines:
+                if (
-                    abort_due_to_max_lines = True
+                    max_lines is not None
+                    and total_line_count + line_count > max_lines
+                    and should_abort_early
+                ):
+                    logger.debug(
+                        f"sphinx-llms-txt: Stopping collection due to size limit. "
+                        f"File {docname} would exceed limit."
+                    )
+                    aborted_due_to_size = True
                     break
 
                 # Double-check this file should be included (not in excluded patterns)
@@ -315,7 +333,9 @@ class LLMSFullManager:
                 )
 
         # Add any remaining files (in alphabetical order) that aren't in the page order
-        if not abort_due_to_max_lines:
+        # Only skip this if we aborted early due to size limits for skip/note actions
+        size_limit_exceeded = max_lines is not None and total_line_count > max_lines
+        if not (size_limit_exceeded and should_abort_early):
             # Get all source files in the _sources directory using configured suffixes
             source_suffixes = self._get_source_suffixes()
             all_source_files = []
@@ -374,8 +394,13 @@ class LLMSFullManager:
                 # Read and process the file
                 content, line_count = self._read_source_file(file_path, docname)
 
-                # Check if adding this file would exceed the maximum line count
+                # Abort early for skip/note actions
-                if max_lines is not None and total_line_count + line_count > max_lines:
+                if (
+                    max_lines is not None
+                    and total_line_count + line_count > max_lines
+                    and should_abort_early
+                ):
+                    aborted_due_to_size = True
                     break
 
                 if content:
@@ -384,23 +409,26 @@ class LLMSFullManager:
                     total_line_count += line_count
 
         # Process code files at the end if configured
-        if not abort_due_to_max_lines:
+        # Only skip this if we aborted early due to size limits for skip/note actions
+        if not (size_limit_exceeded and should_abort_early):
             code_file_parts, processed_file_paths = self._process_code_files()
             code_files_line_count = sum(
                 part.count("\n") + 1 for part in code_file_parts
             )
 
             # Check if adding code files would exceed the maximum line count
-            max_lines = self.config.get("llms_txt_full_max_size")
+            # For "keep" action, we include code files regardless of size
             if (
                 max_lines is not None
                 and total_line_count + code_files_line_count > max_lines
+                and should_abort_early
             ):
                 logger.warning(
                     f"sphinx-llms-txt: Adding code files would exceed max line limit "
                     f"({max_lines}). Current: {total_line_count}, "
                     f"Code files: {code_files_line_count}. Skipping code files."
                 )
+                aborted_due_to_size = True
             else:
                 # Add source code files section if there are any code files
                 if code_file_parts:
@@ -413,30 +441,58 @@ class LLMSFullManager:
                     total_line_count += (
                         code_files_line_count + section_header.count("\n") + 1
                     )
+        else:
+            # If we aborted early for skip/note actions, set empty code file parts
+            code_file_parts = []
 
-        # Check if line limit was exceeded before creating the file
+        # Handle size limit exceeded cases
-        max_lines = self.config.get("llms_txt_full_max_size")
+        if max_lines is not None and (
-        if abort_due_to_max_lines or (
+            total_line_count > max_lines or aborted_due_to_size
-            max_lines is not None and total_line_count > max_lines
         ):
-            logger.warning(
+            # Parse the size_policy configuration (reuse what we parsed earlier)
-                f"sphinx-llms-txt: Max line limit ({max_lines}) exceeded:"
+            size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip")
-                f" {total_line_count} > {max_lines}. "
+            log_level, action = self._parse_size_policy_config(size_policy)
-                f"Not creating llms-full.txt file."
+
+            # Log with the specified level
+            filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
+            message = f"sphinx-llms-txt: Max lines ({max_lines}) exceeded for {filename}"  # noqa: E501
+
+            if log_level == "info":
+                logger.info(message)
+            else:
+                logger.warning(message)
+
+            # Handle different actions
+            if action == "skip":
+                filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
+                logger.info(f"sphinx-llms-txt: Skipping {filename} generation")
+                # Log summary information if requested
+                if self.config.get("llms_txt_file"):
+                    self.writer.write_verbose_info_to_file(
+                        page_order, self.collector.page_titles, total_line_count
+                    )
+                return
+            elif action == "note":
+                logger.info(f"sphinx-llms-txt: Creating placeholder {output_path}")
+                self._write_placeholder_file(output_path, max_lines)
+
+                # Log summary information if requested
+                if self.config.get("llms_txt_file"):
+                    self.writer.write_verbose_info_to_file(
+                        page_order, self.collector.page_titles, total_line_count
+                    )
+                return
+            elif action == "keep":
+                filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
+                # Fall through to write the file
+
+        # Write combined file only if we have content to write
+        if content_parts:
+            success = self.writer.write_combined_file(
+                content_parts, output_path, total_line_count
             )
-
+        else:
-            # Log summary information if requested
+            success = False
-            if self.config.get("llms_txt_file"):
-                self.writer.write_verbose_info_to_file(
-                    page_order, self.collector.page_titles, total_line_count
-                )
-
-            return
-
-        # Write combined file if limit wasn't exceeded
-        success = self.writer.write_combined_file(
-            content_parts, output_path, total_line_count
-        )
 
         # Log summary information if requested
         if success and self.config.get("llms_txt_file"):
@@ -813,3 +869,72 @@ class LLMSFullManager:
             # Recursively handle subdirectories
             if subtree is not None:  # It's a directory
                 self._format_tree_node(subtree, lines, next_prefix, False)
+
+    def _parse_size_policy_config(self, size_policy: str) -> tuple[str, str]:
+        """Parse the llms_txt_full_size_policy configuration value.
+
+        Args:
+            size_policy: Configuration string in format "loglevel_action"
+
+        Returns:
+            Tuple of (log_level, action) where:
+            - log_level is "warn" or "info"
+            - action is "keep", "skip", or "note"
+        """
+        if not size_policy or "_" not in size_policy:
+            logger.warning(
+                f"sphinx-llms-txt: Invalid llms_txt_full_size_policy "
+                f"format: '{size_policy}'. "
+                f"Using default 'warn_skip'."
+            )
+            return "warn", "skip"
+
+        parts = size_policy.split("_", 1)  # Split on first underscore only
+        log_level, action = parts[0], parts[1]
+
+        # Validate log level
+        if log_level not in ["warn", "info"]:
+            logger.warning(
+                f"sphinx-llms-txt: Invalid log level '{log_level}' in "
+                f"llms_txt_full_size_policy. "
+                f"Valid options: warn, info. Using 'warn'."
+            )
+            log_level = "warn"
+
+        # Validate action
+        if action not in ["keep", "skip", "note"]:
+            logger.warning(
+                f"sphinx-llms-txt: Invalid action '{action}' in "
+                f"llms_txt_full_size_policy. "
+                f"Valid options: keep, skip, note. Using 'skip'."
+            )
+            action = "skip"
+
+        return log_level, action
+
+    def _write_placeholder_file(self, output_path: Path, max_lines: int):
+        """Write a placeholder llms-full.txt file with a note about size limit.
+
+        Args:
+            output_path: Path where the placeholder file should be written
+            max_lines: The configured maximum line limit
+        """
+        # Create the placeholder note content
+        placeholder_content = (
+            f".. This file was not generated because it exceeded the configured size limit.\n"  # noqa: E501
+            "   See the conf.py ``llms_txt_full_max_size`` and ``llms_txt_full_size_policy``\n"  # noqa: E501
+            "   for configuration options.\n"
+            "\n"
+            f"   Configured max size: {max_lines} lines\n"
+            "\n"
+            "   For more information, see: https://sphinx-llms-txt.readthedocs.io/en/latest/configuration-values.html#llms-txt-full-max-size\n"  # noqa: E501
+        )
+
+        try:
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write(placeholder_content)
+            logger.debug(f"sphinx-llms-txt: Wrote placeholder file: {output_path}")
+        except Exception as e:
+            logger.error(
+                f"sphinx-llms-txt: Error writing placeholder file {output_path}: {e}"
+            )

sphinx_llms_txt/writer.py

@@ -37,7 +37,7 @@ class FileWriter:
                 f.write("\n".join(content_parts))
 
             logger.info(
-                f"sphinx-llms-txt: created {output_path} with {len(content_parts)}"
+                f"sphinx-llms-txt: Created {output_path} with {len(content_parts)}"
                 f" sources and {total_line_count} lines"
             )
             return True

tests/test_integration.py

@@ -117,6 +117,152 @@ def test_max_lines_limit(temp_dir, rootdir):
         app.docutils_conf_path.unlink()
 
 
+def test_on_exceed_skip(temp_dir, rootdir):
+    """Test that skip action works when size limit is exceeded."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_full_filename": "skip-test.txt",
+            "llms_txt_full_max_size": 20,
+            "llms_txt_full_size_policy": "warn_skip",
+        },
+    )
+
+    app.build()
+
+    # Check that the output file was NOT created
+    output_file = Path(app.outdir) / "skip-test.txt"
+    assert (
+        not output_file.exists()
+    ), f"Output file {output_file} should not exist with skip action"
+
+    # Cleanup
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()
+
+
+def test_on_exceed_keep(temp_dir, rootdir):
+    """Test that keep action works when size limit is exceeded."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_full_filename": "keep-test.txt",
+            "llms_txt_full_max_size": 20,
+            "llms_txt_full_size_policy": "info_keep",
+        },
+    )
+
+    app.build()
+
+    # Check that the output file WAS created despite exceeding limit
+    output_file = Path(app.outdir) / "keep-test.txt"
+    assert (
+        output_file.exists()
+    ), f"Output file {output_file} should exist with keep action"
+
+    # Verify it has content
+    content = output_file.read_text()
+    assert len(content) > 0, "Output file should have content with keep action"
+
+    # Cleanup
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()
+
+
+def test_on_exceed_note(temp_dir, rootdir):
+    """Test that note action works when size limit is exceeded."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_full_filename": "note-test.txt",
+            "llms_txt_full_max_size": 20,
+            "llms_txt_full_size_policy": "warn_note",
+        },
+    )
+
+    app.build()
+
+    # Check that the output file WAS created with placeholder content
+    output_file = Path(app.outdir) / "note-test.txt"
+    assert (
+        output_file.exists()
+    ), f"Output file {output_file} should exist with note action"
+
+    # Verify it has the placeholder content
+    content = output_file.read_text()
+    assert (
+        "This file was not generated because it exceeded the configured size limit."
+        in content
+    )
+    assert "llms_txt_full_max_size" in content
+    assert "llms_txt_full_size_policy" in content
+    assert "Configured max size: 20 lines" in content
+
+    # Cleanup
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()
+
+
+def test_on_exceed_invalid_config(temp_dir, rootdir):
+    """Test behavior with invalid configuration values."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_full_filename": "invalid-test.txt",
+            "llms_txt_full_max_size": 20,
+            "llms_txt_full_size_policy": "invalid_format",  # Invalid config
+        },
+    )
+
+    app.build()
+
+    # Should fall back to default behavior (warn_skip)
+    output_file = Path(app.outdir) / "invalid-test.txt"
+    assert (
+        not output_file.exists()
+    ), f"Output file {output_file} should not exist with invalid config fallback"
+
+    # Cleanup
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()
+
+
 def test_title_override(temp_dir, rootdir):
     """Test that the title override works correctly."""
     from sphinx.testing.util import SphinxTestApp

tests/test_llms_txt.py

@@ -762,6 +762,7 @@ def test_summary_default_uses_first_paragraph():
             llms_txt_full_file = True
             llms_txt_full_filename = "llms-full.txt"
             llms_txt_full_max_size = None
+            llms_txt_full_size_policy = "warn_skip"
             llms_txt_directives = []
             llms_txt_exclude = []
             llms_txt_code_files = []