Xarkam/sphinx-llms-txt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add configuration option

Browse Source
This commit is contained in:
Jared Dillard
2025-05-16 13:41:14 -07:00
parent b51f3b099a
commit 69e251171d
5 changed files with 117 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.rst
+5
View File
@@ -1,6 +1,11 @@
Changelog
=========
0.2.0
-----
- Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size
0.1.0
-----
README.md
+6
View File
@@ -32,6 +32,12 @@ extensions = [
- **Default**: `False`
- **Description**: Whether to include a summary in the build output
### `llms_txt_max_lines`
- **Type**: integer or `None`
- **Default**: `None` (no limit)
- **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes.
## License
MIT License - see LICENSE file for details.
sphinx_llms_txt/__init__.py
+68 -25
View File
@@ -9,7 +9,7 @@ from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util import logging
__version__ = "0.1.0"
__version__ = "0.2.0"
logger = logging.getLogger(__name__)
@@ -163,65 +163,106 @@ class LLMSFullManager:
# Add pages in order
added_files = set()
total_line_count = 0
max_lines = self.config.get("llms_txt_max_lines")
abort_due_to_max_lines = False
for docname in page_order:
if docname in docname_to_file:
file_path = docname_to_file[docname]
content = self._read_source_file(file_path, docname)
content, line_count = self._read_source_file(file_path, docname)
# Check if adding this file would exceed the maximum line count
if max_lines is not None and total_line_count + line_count > max_lines:
abort_due_to_max_lines = True
break
if content:
content_parts.append(content)
added_files.add(file_path.stem)
total_line_count += line_count
else:
logger.warning(f"Source file not found for: {docname}")
logger.warning(f"sphinx-llm-txt: Source file not found for: {docname}")
# Add any remaining files (in alphabetical order)
remaining_files = sorted(
[name for name in txt_files if name not in added_files]
)
if remaining_files:
logger.info(f"Adding remaining files: {remaining_files}")
for file_stem in remaining_files:
file_path = txt_files[file_stem]
content = self._read_source_file(file_path, file_stem)
if content:
content_parts.append(content)
# Add any remaining files (in alphabetical order) if not aborted due to max lines
if not abort_due_to_max_lines:
remaining_files = sorted(
[name for name in txt_files if name not in added_files]
)
if remaining_files:
logger.info(f"Adding remaining files: {remaining_files}")
for file_stem in remaining_files:
file_path = txt_files[file_stem]
content, line_count = self._read_source_file(file_path, file_stem)
# Write combined file
# Check if adding this file would exceed the maximum line count
if max_lines is not None and total_line_count + line_count > max_lines:
break
if content:
content_parts.append(content)
total_line_count += line_count
# Check if line limit was exceeded before creating the file
max_lines = self.config.get("llms_txt_max_lines")
if abort_due_to_max_lines or (max_lines is not None and total_line_count > max_lines):
logger.warning(
f"sphinx-llm-txt: Max line limit ({max_lines}) exceeded: {total_line_count} > {max_lines}. "
f"Not creating llms-full.txt file."
)
# Log summary information if requested
if self.config.get("llms_txt_verbose"):
self._log_summary_info(page_order, total_line_count)
return
# Write combined file if limit wasn't exceeded
try:
with open(output_path, "w", encoding="utf-8") as f:
f.write("\n".join(content_parts))
logger.info(
f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources"
f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources and {total_line_count} lines"
)
# Log summary information if requested
if self.config.get("llms_txt_verbose"):
self._log_summary_info(page_order)
self._log_summary_info(page_order, total_line_count)
except Exception as e:
logger.error(f"Error writing combined sources file: {e}")
logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}")
def _read_source_file(self, file_path: Path, docname: str) -> str:
"""Read and format a single source file."""
def _read_source_file(self, file_path: Path, docname: str) -> tuple:
"""Read and format a single source file.
Returns:
tuple: (content_str, line_count) where line_count is the number of lines in the file
"""
try:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
section_lines = [content, ""]
# Count the lines in the content
line_count = content.count('\n') + (0 if content.endswith('\n') else 1)
return "\n".join(section_lines)
section_lines = [content, ""]
content_str = "\n".join(section_lines)
# Add 2 for the section_lines (content + empty line)
return content_str, line_count + 1
except Exception as e:
logger.error(f"Error reading source file {file_path}: {e}")
return ""
logger.error(f"sphinx-llm-txt: Error reading source file {file_path}: {e}")
return "", 0
def _log_summary_info(self, page_order: List[str]):
def _log_summary_info(self, page_order: List[str], total_line_count: int = 0):
"""Log summary information to the logger."""
logger.info("")
logger.info("llms-txt Summary")
logger.info("================")
logger.info(f"Total pages: {len(page_order)}")
logger.info(f"Configuration: {self.config}")
logger.info("Page order:")
for i, docname in enumerate(page_order, 1):
@@ -259,6 +300,7 @@ def build_finished(app: Sphinx, exception):
config = {
"llms_txt_filename": app.config.llms_txt_filename,
"llms_txt_verbose": app.config.llms_txt_verbose,
"llms_txt_max_lines": app.config.llms_txt_max_lines,
}
_manager.set_config(config)
@@ -279,6 +321,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
# Add configuration options
app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_verbose", False, "env")
app.add_config_value("llms_txt_max_lines", None, "env")
# Connect to Sphinx events
app.connect("doctree-resolved", doctree_resolved)
tests/test_integration.py
+36
View File
@@ -1,6 +1,8 @@
"""Integration tests for sphinx-llms-txt."""
import sys
from pathlib import Path
from sphinx.testing.util import _clean_up_global_state
def test_build_html_with_llms_txt(basic_sphinx_app):
@@ -68,3 +70,37 @@ def test_custom_filename(temp_dir, rootdir):
# Safe unlink that works with older Python versions
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_max_lines_limit(temp_dir, rootdir):
"""Test that the max lines limit works correctly."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
# Create a new test app with a small line limit
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_filename": "limited.txt",
"llms_txt_max_lines": 10, # Set a small limit to trigger the warning
"llms_txt_verbose": True,
},
)
app.build()
# Check that the output file was NOT created (since it would exceed the limit)
output_file = Path(app.outdir) / "limited.txt"
assert not output_file.exists(), f"Output file {output_file} exists but should not when limit is exceeded"
# Custom cleanup to avoid missing_ok issue
sys.path[:] = app._saved_path
_clean_up_global_state()
# Safe unlink
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
tests/test_llms_txt.py
+1
View File
@@ -60,6 +60,7 @@ def test_set_config():
config = {
"llms_txt_filename": "custom.txt",
"llms_txt_verbose": True,
"llms_txt_max_lines": 1000,
}
manager.set_config(config)
assert manager.config == config