From 69e251171d2868714c2c2c197aeba06a1d0c6864 Mon Sep 17 00:00:00 2001 From: Jared Dillard Date: Fri, 16 May 2025 13:41:14 -0700 Subject: [PATCH] Add configuration option --- CHANGELOG.rst | 5 ++ README.md | 8 +++- sphinx_llms_txt/__init__.py | 93 +++++++++++++++++++++++++++---------- tests/test_integration.py | 36 ++++++++++++++ tests/test_llms_txt.py | 1 + 5 files changed, 117 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3ca0df7..51afeb5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +0.2.0 +----- + +- Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size + 0.1.0 ----- diff --git a/README.md b/README.md index c1dbbde..66cefde 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,16 @@ extensions = [ ### `llms_txt_verbose` -- **Type**: boolean +- **Type**: boolean - **Default**: `False` - **Description**: Whether to include a summary in the build output +### `llms_txt_max_lines` + +- **Type**: integer or `None` +- **Default**: `None` (no limit) +- **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes. + ## License MIT License - see LICENSE file for details. diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index 9dc58b1..8c69273 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -9,7 +9,7 @@ from sphinx.application import Sphinx from sphinx.environment import BuildEnvironment from sphinx.util import logging -__version__ = "0.1.0" +__version__ = "0.2.0" logger = logging.getLogger(__name__) @@ -163,65 +163,106 @@ class LLMSFullManager: # Add pages in order added_files = set() + total_line_count = 0 + max_lines = self.config.get("llms_txt_max_lines") + abort_due_to_max_lines = False for docname in page_order: if docname in docname_to_file: file_path = docname_to_file[docname] - content = self._read_source_file(file_path, docname) + content, line_count = self._read_source_file(file_path, docname) + + # Check if adding this file would exceed the maximum line count + if max_lines is not None and total_line_count + line_count > max_lines: + abort_due_to_max_lines = True + break + if content: content_parts.append(content) added_files.add(file_path.stem) + total_line_count += line_count else: - logger.warning(f"Source file not found for: {docname}") + logger.warning(f"sphinx-llm-txt: Source file not found for: {docname}") - # Add any remaining files (in alphabetical order) - remaining_files = sorted( - [name for name in txt_files if name not in added_files] - ) - if remaining_files: - logger.info(f"Adding remaining files: {remaining_files}") - for file_stem in remaining_files: - file_path = txt_files[file_stem] - content = self._read_source_file(file_path, file_stem) - if content: - content_parts.append(content) + # Add any remaining files (in alphabetical order) if not aborted due to max lines + if not abort_due_to_max_lines: + remaining_files = sorted( + [name for name in txt_files if name not in added_files] + ) + if remaining_files: + logger.info(f"Adding remaining files: {remaining_files}") + for file_stem in remaining_files: + file_path = txt_files[file_stem] + content, line_count = self._read_source_file(file_path, file_stem) - # Write combined file + # Check if adding this file would exceed the maximum line count + if max_lines is not None and total_line_count + line_count > max_lines: + break + + if content: + content_parts.append(content) + total_line_count += line_count + + # Check if line limit was exceeded before creating the file + max_lines = self.config.get("llms_txt_max_lines") + if abort_due_to_max_lines or (max_lines is not None and total_line_count > max_lines): + logger.warning( + f"sphinx-llm-txt: Max line limit ({max_lines}) exceeded: {total_line_count} > {max_lines}. " + f"Not creating llms-full.txt file." + ) + + # Log summary information if requested + if self.config.get("llms_txt_verbose"): + self._log_summary_info(page_order, total_line_count) + + return + + # Write combined file if limit wasn't exceeded try: with open(output_path, "w", encoding="utf-8") as f: f.write("\n".join(content_parts)) logger.info( - f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources" + f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources and {total_line_count} lines" ) # Log summary information if requested if self.config.get("llms_txt_verbose"): - self._log_summary_info(page_order) + self._log_summary_info(page_order, total_line_count) except Exception as e: - logger.error(f"Error writing combined sources file: {e}") + logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}") - def _read_source_file(self, file_path: Path, docname: str) -> str: - """Read and format a single source file.""" + def _read_source_file(self, file_path: Path, docname: str) -> tuple: + """Read and format a single source file. + + Returns: + tuple: (content_str, line_count) where line_count is the number of lines in the file + """ try: with open(file_path, "r", encoding="utf-8") as f: content = f.read() - section_lines = [content, ""] + # Count the lines in the content + line_count = content.count('\n') + (0 if content.endswith('\n') else 1) - return "\n".join(section_lines) + section_lines = [content, ""] + content_str = "\n".join(section_lines) + + # Add 2 for the section_lines (content + empty line) + return content_str, line_count + 1 except Exception as e: - logger.error(f"Error reading source file {file_path}: {e}") - return "" + logger.error(f"sphinx-llm-txt: Error reading source file {file_path}: {e}") + return "", 0 - def _log_summary_info(self, page_order: List[str]): + def _log_summary_info(self, page_order: List[str], total_line_count: int = 0): """Log summary information to the logger.""" logger.info("") logger.info("llms-txt Summary") logger.info("================") logger.info(f"Total pages: {len(page_order)}") + logger.info(f"Configuration: {self.config}") logger.info("Page order:") for i, docname in enumerate(page_order, 1): @@ -259,6 +300,7 @@ def build_finished(app: Sphinx, exception): config = { "llms_txt_filename": app.config.llms_txt_filename, "llms_txt_verbose": app.config.llms_txt_verbose, + "llms_txt_max_lines": app.config.llms_txt_max_lines, } _manager.set_config(config) @@ -279,6 +321,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: # Add configuration options app.add_config_value("llms_txt_filename", "llms-full.txt", "env") app.add_config_value("llms_txt_verbose", False, "env") + app.add_config_value("llms_txt_max_lines", None, "env") # Connect to Sphinx events app.connect("doctree-resolved", doctree_resolved) diff --git a/tests/test_integration.py b/tests/test_integration.py index 0440b56..8cba01f 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,6 +1,8 @@ """Integration tests for sphinx-llms-txt.""" +import sys from pathlib import Path +from sphinx.testing.util import _clean_up_global_state def test_build_html_with_llms_txt(basic_sphinx_app): @@ -68,3 +70,37 @@ def test_custom_filename(temp_dir, rootdir): # Safe unlink that works with older Python versions if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): app.docutils_conf_path.unlink() + + +def test_max_lines_limit(temp_dir, rootdir): + """Test that the max lines limit works correctly.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + # Create a new test app with a small line limit + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_filename": "limited.txt", + "llms_txt_max_lines": 10, # Set a small limit to trigger the warning + "llms_txt_verbose": True, + }, + ) + + app.build() + + # Check that the output file was NOT created (since it would exceed the limit) + output_file = Path(app.outdir) / "limited.txt" + assert not output_file.exists(), f"Output file {output_file} exists but should not when limit is exceeded" + + # Custom cleanup to avoid missing_ok issue + sys.path[:] = app._saved_path + _clean_up_global_state() + + # Safe unlink + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index bc71f75..91bb2cd 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -60,6 +60,7 @@ def test_set_config(): config = { "llms_txt_filename": "custom.txt", "llms_txt_verbose": True, + "llms_txt_max_lines": 1000, } manager.set_config(config) assert manager.config == config