diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8a555b..ea56c66 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +0.5.0 +----- + +- Add :confval:`llms_txt_full_size_policy` configuration option to control behavior when :confval:`llms_txt_full_max_size` is exceeded. + `#35 `_ + 0.4.1 ----- diff --git a/docs/source/advanced-configuration.rst b/docs/source/advanced-configuration.rst index 714d23a..403b7dc 100644 --- a/docs/source/advanced-configuration.rst +++ b/docs/source/advanced-configuration.rst @@ -76,15 +76,26 @@ Handling Large Documentation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For very large documentation sets, generating the full documentation file might exceed reasonable size limits. -You can set a maximum line count: +You can set a maximum line count and control what happens when that limit is exceeded: .. code-block:: python llms_txt_full_max_size = 10000 # Maximum 10,000 lines + llms_txt_full_size_policy = "warn_skip" # Default behavior -If the generated file would exceed this limit, the extension will skip its generation and show a warning, allowing the build to complete. +The ``llms_txt_full_size_policy`` setting controls both the log level and action taken when the size limit is exceeded. +It uses the format ``"_"``: -.. tip:: Use :ref:`excluding_content` to remove less relevant pages. +**Log levels:** +- ``warn``: Log as a warning (default) +- ``info``: Log as informational message + +**Actions:** +- ``skip``: Don't create the file (default) +- ``keep``: Create the file anyway, ignoring the size limit +- ``note``: Create a placeholder file explaining why the full file wasn't generated + +.. tip:: Use :ref:`excluding_content` to remove less relevant pages and reduce the file size. .. _custom_directive_handling: @@ -198,6 +209,7 @@ Here's a complete example showing multiple :doc:`configuration-values`: llms_txt_filename = "ai-summary.txt" llms_txt_full_filename = "ai-full-docs.txt" llms_txt_full_max_size = 50000 + llms_txt_full_size_policy = "warn_note" # Content customization llms_txt_title = "Project Documentation for AI Assistants" diff --git a/docs/source/configuration-values.rst b/docs/source/configuration-values.rst index 4009e35..5e8904d 100644 --- a/docs/source/configuration-values.rst +++ b/docs/source/configuration-values.rst @@ -24,11 +24,22 @@ Project Configuration Values - **Type**: integer or ``None`` - **Default**: ``None`` (no limit) - **Description**: Sets a maximum line count for ``llms_txt_full_filename``. - If exceeded, the file is skipped and a warning is shown, but the build still completes. + Behavior when exceeded is controlled by :confval:`llms_txt_full_size_policy`. See :ref:`handling_large_documentation`. .. versionadded:: 0.2.0 +.. confval:: llms_txt_full_size_policy + + - **Type**: string + - **Default**: ``'warn_skip'`` + - **Description**: Controls what happens when :confval:`llms_txt_full_max_size` is exceeded. + Format is ``_``. Log levels: ``warn``, ``info``. + Actions: ``skip``, ``keep``, ``note``. + See :ref:`handling_large_documentation`. + + .. versionadded:: 0.5.0 + .. confval:: llms_txt_file - **Type**: boolean diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index 8f43be5..e201b97 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -12,7 +12,7 @@ from .manager import LLMSFullManager from .processor import DocumentProcessor from .writer import FileWriter -__version__ = "0.4.1" +__version__ = "0.5.0" # Export classes needed by tests __all__ = [ @@ -74,6 +74,7 @@ def build_finished(app: Sphinx, exception): "llms_txt_full_file": app.config.llms_txt_full_file, "llms_txt_full_filename": app.config.llms_txt_full_filename, "llms_txt_full_max_size": app.config.llms_txt_full_max_size, + "llms_txt_full_size_policy": app.config.llms_txt_full_size_policy, "llms_txt_directives": app.config.llms_txt_directives, "llms_txt_exclude": app.config.llms_txt_exclude, "llms_txt_code_files": app.config.llms_txt_code_files, @@ -102,6 +103,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_config_value("llms_txt_full_file", True, "env") app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env") app.add_config_value("llms_txt_full_max_size", None, "env") + app.add_config_value("llms_txt_full_size_policy", "warn_skip", "env") app.add_config_value("llms_txt_directives", [], "env") app.add_config_value("llms_txt_title", None, "env") app.add_config_value("llms_txt_summary", None, "env") diff --git a/sphinx_llms_txt/manager.py b/sphinx_llms_txt/manager.py index 9f154e2..b31dcea 100644 --- a/sphinx_llms_txt/manager.py +++ b/sphinx_llms_txt/manager.py @@ -273,16 +273,34 @@ class LLMSFullManager: added_files = set() total_line_count = code_files_line_count max_lines = self.config.get("llms_txt_full_max_size") - abort_due_to_max_lines = False + + # Parse size_policy configuration early to determine collection strategy + size_policy_action = None + aborted_due_to_size = False + if max_lines is not None: + size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip") + _, size_policy_action = self._parse_size_policy_config(size_policy) + + # Only collect all files if action is "keep" + # For "skip" and "note", we can abort early when size limit is exceeded + should_abort_early = size_policy_action in ["skip", "note"] for docname, _ in page_order: if docname in docname_to_file: file_path = docname_to_file[docname] content, line_count = self._read_source_file(file_path, docname) - # Check if adding this file would exceed the maximum line count - if max_lines is not None and total_line_count + line_count > max_lines: - abort_due_to_max_lines = True + # Abort early for skip/note actions + if ( + max_lines is not None + and total_line_count + line_count > max_lines + and should_abort_early + ): + logger.debug( + f"sphinx-llms-txt: Stopping collection due to size limit. " + f"File {docname} would exceed limit." + ) + aborted_due_to_size = True break # Double-check this file should be included (not in excluded patterns) @@ -315,7 +333,9 @@ class LLMSFullManager: ) # Add any remaining files (in alphabetical order) that aren't in the page order - if not abort_due_to_max_lines: + # Only skip this if we aborted early due to size limits for skip/note actions + size_limit_exceeded = max_lines is not None and total_line_count > max_lines + if not (size_limit_exceeded and should_abort_early): # Get all source files in the _sources directory using configured suffixes source_suffixes = self._get_source_suffixes() all_source_files = [] @@ -374,8 +394,13 @@ class LLMSFullManager: # Read and process the file content, line_count = self._read_source_file(file_path, docname) - # Check if adding this file would exceed the maximum line count - if max_lines is not None and total_line_count + line_count > max_lines: + # Abort early for skip/note actions + if ( + max_lines is not None + and total_line_count + line_count > max_lines + and should_abort_early + ): + aborted_due_to_size = True break if content: @@ -384,23 +409,26 @@ class LLMSFullManager: total_line_count += line_count # Process code files at the end if configured - if not abort_due_to_max_lines: + # Only skip this if we aborted early due to size limits for skip/note actions + if not (size_limit_exceeded and should_abort_early): code_file_parts, processed_file_paths = self._process_code_files() code_files_line_count = sum( part.count("\n") + 1 for part in code_file_parts ) # Check if adding code files would exceed the maximum line count - max_lines = self.config.get("llms_txt_full_max_size") + # For "keep" action, we include code files regardless of size if ( max_lines is not None and total_line_count + code_files_line_count > max_lines + and should_abort_early ): logger.warning( f"sphinx-llms-txt: Adding code files would exceed max line limit " f"({max_lines}). Current: {total_line_count}, " f"Code files: {code_files_line_count}. Skipping code files." ) + aborted_due_to_size = True else: # Add source code files section if there are any code files if code_file_parts: @@ -413,30 +441,58 @@ class LLMSFullManager: total_line_count += ( code_files_line_count + section_header.count("\n") + 1 ) + else: + # If we aborted early for skip/note actions, set empty code file parts + code_file_parts = [] - # Check if line limit was exceeded before creating the file - max_lines = self.config.get("llms_txt_full_max_size") - if abort_due_to_max_lines or ( - max_lines is not None and total_line_count > max_lines + # Handle size limit exceeded cases + if max_lines is not None and ( + total_line_count > max_lines or aborted_due_to_size ): - logger.warning( - f"sphinx-llms-txt: Max line limit ({max_lines}) exceeded:" - f" {total_line_count} > {max_lines}. " - f"Not creating llms-full.txt file." + # Parse the size_policy configuration (reuse what we parsed earlier) + size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip") + log_level, action = self._parse_size_policy_config(size_policy) + + # Log with the specified level + filename = self.config.get("llms_txt_full_filename", "llms-full.txt") + message = f"sphinx-llms-txt: Max lines ({max_lines}) exceeded for {filename}" # noqa: E501 + + if log_level == "info": + logger.info(message) + else: + logger.warning(message) + + # Handle different actions + if action == "skip": + filename = self.config.get("llms_txt_full_filename", "llms-full.txt") + logger.info(f"sphinx-llms-txt: Skipping {filename} generation") + # Log summary information if requested + if self.config.get("llms_txt_file"): + self.writer.write_verbose_info_to_file( + page_order, self.collector.page_titles, total_line_count + ) + return + elif action == "note": + logger.info(f"sphinx-llms-txt: Creating placeholder {output_path}") + self._write_placeholder_file(output_path, max_lines) + + # Log summary information if requested + if self.config.get("llms_txt_file"): + self.writer.write_verbose_info_to_file( + page_order, self.collector.page_titles, total_line_count + ) + return + elif action == "keep": + filename = self.config.get("llms_txt_full_filename", "llms-full.txt") + # Fall through to write the file + + # Write combined file only if we have content to write + if content_parts: + success = self.writer.write_combined_file( + content_parts, output_path, total_line_count ) - - # Log summary information if requested - if self.config.get("llms_txt_file"): - self.writer.write_verbose_info_to_file( - page_order, self.collector.page_titles, total_line_count - ) - - return - - # Write combined file if limit wasn't exceeded - success = self.writer.write_combined_file( - content_parts, output_path, total_line_count - ) + else: + success = False # Log summary information if requested if success and self.config.get("llms_txt_file"): @@ -813,3 +869,72 @@ class LLMSFullManager: # Recursively handle subdirectories if subtree is not None: # It's a directory self._format_tree_node(subtree, lines, next_prefix, False) + + def _parse_size_policy_config(self, size_policy: str) -> tuple[str, str]: + """Parse the llms_txt_full_size_policy configuration value. + + Args: + size_policy: Configuration string in format "loglevel_action" + + Returns: + Tuple of (log_level, action) where: + - log_level is "warn" or "info" + - action is "keep", "skip", or "note" + """ + if not size_policy or "_" not in size_policy: + logger.warning( + f"sphinx-llms-txt: Invalid llms_txt_full_size_policy " + f"format: '{size_policy}'. " + f"Using default 'warn_skip'." + ) + return "warn", "skip" + + parts = size_policy.split("_", 1) # Split on first underscore only + log_level, action = parts[0], parts[1] + + # Validate log level + if log_level not in ["warn", "info"]: + logger.warning( + f"sphinx-llms-txt: Invalid log level '{log_level}' in " + f"llms_txt_full_size_policy. " + f"Valid options: warn, info. Using 'warn'." + ) + log_level = "warn" + + # Validate action + if action not in ["keep", "skip", "note"]: + logger.warning( + f"sphinx-llms-txt: Invalid action '{action}' in " + f"llms_txt_full_size_policy. " + f"Valid options: keep, skip, note. Using 'skip'." + ) + action = "skip" + + return log_level, action + + def _write_placeholder_file(self, output_path: Path, max_lines: int): + """Write a placeholder llms-full.txt file with a note about size limit. + + Args: + output_path: Path where the placeholder file should be written + max_lines: The configured maximum line limit + """ + # Create the placeholder note content + placeholder_content = ( + f".. This file was not generated because it exceeded the configured size limit.\n" # noqa: E501 + " See the conf.py ``llms_txt_full_max_size`` and ``llms_txt_full_size_policy``\n" # noqa: E501 + " for configuration options.\n" + "\n" + f" Configured max size: {max_lines} lines\n" + "\n" + " For more information, see: https://sphinx-llms-txt.readthedocs.io/en/latest/configuration-values.html#llms-txt-full-max-size\n" # noqa: E501 + ) + + try: + with open(output_path, "w", encoding="utf-8") as f: + f.write(placeholder_content) + logger.debug(f"sphinx-llms-txt: Wrote placeholder file: {output_path}") + except Exception as e: + logger.error( + f"sphinx-llms-txt: Error writing placeholder file {output_path}: {e}" + ) diff --git a/sphinx_llms_txt/writer.py b/sphinx_llms_txt/writer.py index 721f391..e5f873b 100644 --- a/sphinx_llms_txt/writer.py +++ b/sphinx_llms_txt/writer.py @@ -37,7 +37,7 @@ class FileWriter: f.write("\n".join(content_parts)) logger.info( - f"sphinx-llms-txt: created {output_path} with {len(content_parts)}" + f"sphinx-llms-txt: Created {output_path} with {len(content_parts)}" f" sources and {total_line_count} lines" ) return True diff --git a/tests/test_integration.py b/tests/test_integration.py index c945d3a..f93e531 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -117,6 +117,152 @@ def test_max_lines_limit(temp_dir, rootdir): app.docutils_conf_path.unlink() +def test_on_exceed_skip(temp_dir, rootdir): + """Test that skip action works when size limit is exceeded.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_full_filename": "skip-test.txt", + "llms_txt_full_max_size": 20, + "llms_txt_full_size_policy": "warn_skip", + }, + ) + + app.build() + + # Check that the output file was NOT created + output_file = Path(app.outdir) / "skip-test.txt" + assert ( + not output_file.exists() + ), f"Output file {output_file} should not exist with skip action" + + # Cleanup + sys.path[:] = app._saved_path + _clean_up_global_state() + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() + + +def test_on_exceed_keep(temp_dir, rootdir): + """Test that keep action works when size limit is exceeded.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_full_filename": "keep-test.txt", + "llms_txt_full_max_size": 20, + "llms_txt_full_size_policy": "info_keep", + }, + ) + + app.build() + + # Check that the output file WAS created despite exceeding limit + output_file = Path(app.outdir) / "keep-test.txt" + assert ( + output_file.exists() + ), f"Output file {output_file} should exist with keep action" + + # Verify it has content + content = output_file.read_text() + assert len(content) > 0, "Output file should have content with keep action" + + # Cleanup + sys.path[:] = app._saved_path + _clean_up_global_state() + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() + + +def test_on_exceed_note(temp_dir, rootdir): + """Test that note action works when size limit is exceeded.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_full_filename": "note-test.txt", + "llms_txt_full_max_size": 20, + "llms_txt_full_size_policy": "warn_note", + }, + ) + + app.build() + + # Check that the output file WAS created with placeholder content + output_file = Path(app.outdir) / "note-test.txt" + assert ( + output_file.exists() + ), f"Output file {output_file} should exist with note action" + + # Verify it has the placeholder content + content = output_file.read_text() + assert ( + "This file was not generated because it exceeded the configured size limit." + in content + ) + assert "llms_txt_full_max_size" in content + assert "llms_txt_full_size_policy" in content + assert "Configured max size: 20 lines" in content + + # Cleanup + sys.path[:] = app._saved_path + _clean_up_global_state() + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() + + +def test_on_exceed_invalid_config(temp_dir, rootdir): + """Test behavior with invalid configuration values.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_full_filename": "invalid-test.txt", + "llms_txt_full_max_size": 20, + "llms_txt_full_size_policy": "invalid_format", # Invalid config + }, + ) + + app.build() + + # Should fall back to default behavior (warn_skip) + output_file = Path(app.outdir) / "invalid-test.txt" + assert ( + not output_file.exists() + ), f"Output file {output_file} should not exist with invalid config fallback" + + # Cleanup + sys.path[:] = app._saved_path + _clean_up_global_state() + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() + + def test_title_override(temp_dir, rootdir): """Test that the title override works correctly.""" from sphinx.testing.util import SphinxTestApp diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index 44487b6..559997e 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -762,6 +762,7 @@ def test_summary_default_uses_first_paragraph(): llms_txt_full_file = True llms_txt_full_filename = "llms-full.txt" llms_txt_full_max_size = None + llms_txt_full_size_policy = "warn_skip" llms_txt_directives = [] llms_txt_exclude = [] llms_txt_code_files = []