Xarkam/sphinx-llms-txt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Allow configuration for behavior when llms-full.txt is too large (#35)

Browse Source
This commit is contained in:
Jared Dillard
2025-08-08 22:00:30 -07:00
committed by GitHub
parent c24f92031c
commit b4cab5ab52
8 changed files with 339 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.rst
+6
View File
@@ -1,6 +1,12 @@
Changelog
=========
0.5.0
-----
- Add :confval:`llms_txt_full_size_policy` configuration option to control behavior when :confval:`llms_txt_full_max_size` is exceeded.
`#35 <https://github.com/jdillard/sphinx-llms-txt/pull/35>`_
0.4.1
-----
docs/source/advanced-configuration.rst
+15 -3
View File
@@ -76,15 +76,26 @@ Handling Large Documentation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
For very large documentation sets, generating the full documentation file might exceed reasonable size limits.
You can set a maximum line count:
You can set a maximum line count and control what happens when that limit is exceeded:
.. code-block:: python
llms_txt_full_max_size = 10000 # Maximum 10,000 lines
llms_txt_full_size_policy = "warn_skip" # Default behavior
If the generated file would exceed this limit, the extension will skip its generation and show a warning, allowing the build to complete.
The ``llms_txt_full_size_policy`` setting controls both the log level and action taken when the size limit is exceeded.
It uses the format ``"<loglevel>_<action>"``:
.. tip:: Use :ref:`excluding_content` to remove less relevant pages.
**Log levels:**
- ``warn``: Log as a warning (default)
- ``info``: Log as informational message
**Actions:**
- ``skip``: Don't create the file (default)
- ``keep``: Create the file anyway, ignoring the size limit
- ``note``: Create a placeholder file explaining why the full file wasn't generated
.. tip:: Use :ref:`excluding_content` to remove less relevant pages and reduce the file size.
.. _custom_directive_handling:
@@ -198,6 +209,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
llms_txt_filename = "ai-summary.txt"
llms_txt_full_filename = "ai-full-docs.txt"
llms_txt_full_max_size = 50000
llms_txt_full_size_policy = "warn_note"
# Content customization
llms_txt_title = "Project Documentation for AI Assistants"
docs/source/configuration-values.rst
+12 -1
View File
@@ -24,11 +24,22 @@ Project Configuration Values
- **Type**: integer or ``None``
- **Default**: ``None`` (no limit)
- **Description**: Sets a maximum line count for ``llms_txt_full_filename``.
If exceeded, the file is skipped and a warning is shown, but the build still completes.
Behavior when exceeded is controlled by :confval:`llms_txt_full_size_policy`.
See :ref:`handling_large_documentation`.
.. versionadded:: 0.2.0
.. confval:: llms_txt_full_size_policy
- **Type**: string
- **Default**: ``'warn_skip'``
- **Description**: Controls what happens when :confval:`llms_txt_full_max_size` is exceeded.
Format is ``<loglevel>_<action>``. Log levels: ``warn``, ``info``.
Actions: ``skip``, ``keep``, ``note``.
See :ref:`handling_large_documentation`.
.. versionadded:: 0.5.0
.. confval:: llms_txt_file
- **Type**: boolean
sphinx_llms_txt/__init__.py
+3 -1
View File
@@ -12,7 +12,7 @@ from .manager import LLMSFullManager
from .processor import DocumentProcessor
from .writer import FileWriter
__version__ = "0.4.1"
__version__ = "0.5.0"
# Export classes needed by tests
__all__ = [
@@ -74,6 +74,7 @@ def build_finished(app: Sphinx, exception):
"llms_txt_full_file": app.config.llms_txt_full_file,
"llms_txt_full_filename": app.config.llms_txt_full_filename,
"llms_txt_full_max_size": app.config.llms_txt_full_max_size,
"llms_txt_full_size_policy": app.config.llms_txt_full_size_policy,
"llms_txt_directives": app.config.llms_txt_directives,
"llms_txt_exclude": app.config.llms_txt_exclude,
"llms_txt_code_files": app.config.llms_txt_code_files,
@@ -102,6 +103,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
app.add_config_value("llms_txt_full_file", True, "env")
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_full_max_size", None, "env")
app.add_config_value("llms_txt_full_size_policy", "warn_skip", "env")
app.add_config_value("llms_txt_directives", [], "env")
app.add_config_value("llms_txt_title", None, "env")
app.add_config_value("llms_txt_summary", None, "env")
sphinx_llms_txt/manager.py
+155 -30
View File
@@ -273,16 +273,34 @@ class LLMSFullManager:
added_files = set()
total_line_count = code_files_line_count
max_lines = self.config.get("llms_txt_full_max_size")
abort_due_to_max_lines = False
# Parse size_policy configuration early to determine collection strategy
size_policy_action = None
aborted_due_to_size = False
if max_lines is not None:
size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip")
_, size_policy_action = self._parse_size_policy_config(size_policy)
# Only collect all files if action is "keep"
# For "skip" and "note", we can abort early when size limit is exceeded
should_abort_early = size_policy_action in ["skip", "note"]
for docname, _ in page_order:
if docname in docname_to_file:
file_path = docname_to_file[docname]
content, line_count = self._read_source_file(file_path, docname)
# Check if adding this file would exceed the maximum line count
if max_lines is not None and total_line_count + line_count > max_lines:
abort_due_to_max_lines = True
# Abort early for skip/note actions
if (
max_lines is not None
and total_line_count + line_count > max_lines
and should_abort_early
):
logger.debug(
f"sphinx-llms-txt: Stopping collection due to size limit. "
f"File {docname} would exceed limit."
)
aborted_due_to_size = True
break
# Double-check this file should be included (not in excluded patterns)
@@ -315,7 +333,9 @@ class LLMSFullManager:
)
# Add any remaining files (in alphabetical order) that aren't in the page order
if not abort_due_to_max_lines:
# Only skip this if we aborted early due to size limits for skip/note actions
size_limit_exceeded = max_lines is not None and total_line_count > max_lines
if not (size_limit_exceeded and should_abort_early):
# Get all source files in the _sources directory using configured suffixes
source_suffixes = self._get_source_suffixes()
all_source_files = []
@@ -374,8 +394,13 @@ class LLMSFullManager:
# Read and process the file
content, line_count = self._read_source_file(file_path, docname)
# Check if adding this file would exceed the maximum line count
if max_lines is not None and total_line_count + line_count > max_lines:
# Abort early for skip/note actions
if (
max_lines is not None
and total_line_count + line_count > max_lines
and should_abort_early
):
aborted_due_to_size = True
break
if content:
@@ -384,23 +409,26 @@ class LLMSFullManager:
total_line_count += line_count
# Process code files at the end if configured
if not abort_due_to_max_lines:
# Only skip this if we aborted early due to size limits for skip/note actions
if not (size_limit_exceeded and should_abort_early):
code_file_parts, processed_file_paths = self._process_code_files()
code_files_line_count = sum(
part.count("\n") + 1 for part in code_file_parts
)
# Check if adding code files would exceed the maximum line count
max_lines = self.config.get("llms_txt_full_max_size")
# For "keep" action, we include code files regardless of size
if (
max_lines is not None
and total_line_count + code_files_line_count > max_lines
and should_abort_early
):
logger.warning(
f"sphinx-llms-txt: Adding code files would exceed max line limit "
f"({max_lines}). Current: {total_line_count}, "
f"Code files: {code_files_line_count}. Skipping code files."
)
aborted_due_to_size = True
else:
# Add source code files section if there are any code files
if code_file_parts:
@@ -413,30 +441,58 @@ class LLMSFullManager:
total_line_count += (
code_files_line_count + section_header.count("\n") + 1
)
else:
# If we aborted early for skip/note actions, set empty code file parts
code_file_parts = []
# Check if line limit was exceeded before creating the file
max_lines = self.config.get("llms_txt_full_max_size")
if abort_due_to_max_lines or (
max_lines is not None and total_line_count > max_lines
# Handle size limit exceeded cases
if max_lines is not None and (
total_line_count > max_lines or aborted_due_to_size
):
logger.warning(
f"sphinx-llms-txt: Max line limit ({max_lines}) exceeded:"
f" {total_line_count} > {max_lines}. "
f"Not creating llms-full.txt file."
# Parse the size_policy configuration (reuse what we parsed earlier)
size_policy = self.config.get("llms_txt_full_size_policy", "warn_skip")
log_level, action = self._parse_size_policy_config(size_policy)
# Log with the specified level
filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
message = f"sphinx-llms-txt: Max lines ({max_lines}) exceeded for {filename}" # noqa: E501
if log_level == "info":
logger.info(message)
else:
logger.warning(message)
# Handle different actions
if action == "skip":
filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
logger.info(f"sphinx-llms-txt: Skipping {filename} generation")
# Log summary information if requested
if self.config.get("llms_txt_file"):
self.writer.write_verbose_info_to_file(
page_order, self.collector.page_titles, total_line_count
)
return
elif action == "note":
logger.info(f"sphinx-llms-txt: Creating placeholder {output_path}")
self._write_placeholder_file(output_path, max_lines)
# Log summary information if requested
if self.config.get("llms_txt_file"):
self.writer.write_verbose_info_to_file(
page_order, self.collector.page_titles, total_line_count
)
return
elif action == "keep":
filename = self.config.get("llms_txt_full_filename", "llms-full.txt")
# Fall through to write the file
# Write combined file only if we have content to write
if content_parts:
success = self.writer.write_combined_file(
content_parts, output_path, total_line_count
)
# Log summary information if requested
if self.config.get("llms_txt_file"):
self.writer.write_verbose_info_to_file(
page_order, self.collector.page_titles, total_line_count
)
return
# Write combined file if limit wasn't exceeded
success = self.writer.write_combined_file(
content_parts, output_path, total_line_count
)
else:
success = False
# Log summary information if requested
if success and self.config.get("llms_txt_file"):
@@ -813,3 +869,72 @@ class LLMSFullManager:
# Recursively handle subdirectories
if subtree is not None: # It's a directory
self._format_tree_node(subtree, lines, next_prefix, False)
def _parse_size_policy_config(self, size_policy: str) -> tuple[str, str]:
"""Parse the llms_txt_full_size_policy configuration value.
Args:
size_policy: Configuration string in format "loglevel_action"
Returns:
Tuple of (log_level, action) where:
- log_level is "warn" or "info"
- action is "keep", "skip", or "note"
"""
if not size_policy or "_" not in size_policy:
logger.warning(
f"sphinx-llms-txt: Invalid llms_txt_full_size_policy "
f"format: '{size_policy}'. "
f"Using default 'warn_skip'."
)
return "warn", "skip"
parts = size_policy.split("_", 1) # Split on first underscore only
log_level, action = parts[0], parts[1]
# Validate log level
if log_level not in ["warn", "info"]:
logger.warning(
f"sphinx-llms-txt: Invalid log level '{log_level}' in "
f"llms_txt_full_size_policy. "
f"Valid options: warn, info. Using 'warn'."
)
log_level = "warn"
# Validate action
if action not in ["keep", "skip", "note"]:
logger.warning(
f"sphinx-llms-txt: Invalid action '{action}' in "
f"llms_txt_full_size_policy. "
f"Valid options: keep, skip, note. Using 'skip'."
)
action = "skip"
return log_level, action
def _write_placeholder_file(self, output_path: Path, max_lines: int):
"""Write a placeholder llms-full.txt file with a note about size limit.
Args:
output_path: Path where the placeholder file should be written
max_lines: The configured maximum line limit
"""
# Create the placeholder note content
placeholder_content = (
f".. This file was not generated because it exceeded the configured size limit.\n" # noqa: E501
" See the conf.py ``llms_txt_full_max_size`` and ``llms_txt_full_size_policy``\n" # noqa: E501
" for configuration options.\n"
"\n"
f" Configured max size: {max_lines} lines\n"
"\n"
" For more information, see: https://sphinx-llms-txt.readthedocs.io/en/latest/configuration-values.html#llms-txt-full-max-size\n" # noqa: E501
)
try:
with open(output_path, "w", encoding="utf-8") as f:
f.write(placeholder_content)
logger.debug(f"sphinx-llms-txt: Wrote placeholder file: {output_path}")
except Exception as e:
logger.error(
f"sphinx-llms-txt: Error writing placeholder file {output_path}: {e}"
)
sphinx_llms_txt/writer.py
+1 -1
View File
@@ -37,7 +37,7 @@ class FileWriter:
f.write("\n".join(content_parts))
logger.info(
f"sphinx-llms-txt: created {output_path} with {len(content_parts)}"
f"sphinx-llms-txt: Created {output_path} with {len(content_parts)}"
f" sources and {total_line_count} lines"
)
return True
tests/test_integration.py
+146
View File
@@ -117,6 +117,152 @@ def test_max_lines_limit(temp_dir, rootdir):
app.docutils_conf_path.unlink()
def test_on_exceed_skip(temp_dir, rootdir):
"""Test that skip action works when size limit is exceeded."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_full_filename": "skip-test.txt",
"llms_txt_full_max_size": 20,
"llms_txt_full_size_policy": "warn_skip",
},
)
app.build()
# Check that the output file was NOT created
output_file = Path(app.outdir) / "skip-test.txt"
assert (
not output_file.exists()
), f"Output file {output_file} should not exist with skip action"
# Cleanup
sys.path[:] = app._saved_path
_clean_up_global_state()
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_on_exceed_keep(temp_dir, rootdir):
"""Test that keep action works when size limit is exceeded."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_full_filename": "keep-test.txt",
"llms_txt_full_max_size": 20,
"llms_txt_full_size_policy": "info_keep",
},
)
app.build()
# Check that the output file WAS created despite exceeding limit
output_file = Path(app.outdir) / "keep-test.txt"
assert (
output_file.exists()
), f"Output file {output_file} should exist with keep action"
# Verify it has content
content = output_file.read_text()
assert len(content) > 0, "Output file should have content with keep action"
# Cleanup
sys.path[:] = app._saved_path
_clean_up_global_state()
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_on_exceed_note(temp_dir, rootdir):
"""Test that note action works when size limit is exceeded."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_full_filename": "note-test.txt",
"llms_txt_full_max_size": 20,
"llms_txt_full_size_policy": "warn_note",
},
)
app.build()
# Check that the output file WAS created with placeholder content
output_file = Path(app.outdir) / "note-test.txt"
assert (
output_file.exists()
), f"Output file {output_file} should exist with note action"
# Verify it has the placeholder content
content = output_file.read_text()
assert (
"This file was not generated because it exceeded the configured size limit."
in content
)
assert "llms_txt_full_max_size" in content
assert "llms_txt_full_size_policy" in content
assert "Configured max size: 20 lines" in content
# Cleanup
sys.path[:] = app._saved_path
_clean_up_global_state()
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_on_exceed_invalid_config(temp_dir, rootdir):
"""Test behavior with invalid configuration values."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_full_filename": "invalid-test.txt",
"llms_txt_full_max_size": 20,
"llms_txt_full_size_policy": "invalid_format", # Invalid config
},
)
app.build()
# Should fall back to default behavior (warn_skip)
output_file = Path(app.outdir) / "invalid-test.txt"
assert (
not output_file.exists()
), f"Output file {output_file} should not exist with invalid config fallback"
# Cleanup
sys.path[:] = app._saved_path
_clean_up_global_state()
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_title_override(temp_dir, rootdir):
"""Test that the title override works correctly."""
from sphinx.testing.util import SphinxTestApp
tests/test_llms_txt.py
+1
View File
@@ -762,6 +762,7 @@ def test_summary_default_uses_first_paragraph():
llms_txt_full_file = True
llms_txt_full_filename = "llms-full.txt"
llms_txt_full_max_size = None
llms_txt_full_size_policy = "warn_skip"
llms_txt_directives = []
llms_txt_exclude = []
llms_txt_code_files = []