Xarkam/sphinx-llms-txt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add llms.txt config options (#9)

Browse Source
This commit is contained in:
Jared Dillard
2025-05-17 20:22:46 -07:00
committed by GitHub
parent 94a82dc68b
commit 10e2b9a684
7 changed files with 210 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.rst
+4 -3
View File
@@ -4,9 +4,10 @@ Changelog
0.2.0
-----
- Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size
- Automatically add content from `.. include::` directives
- Add path resolution for certain directives
- Add `llms_txt_full_max_size` configuration option to limit `llms-full.txt` file size
- Automatically add content from `include` directives in `llms-full.txt`
- Add path resolution for a given set of directives in `llms-full.txt`
- Add `llms.txt` file option, with `llms_txt_title` and `llms_txt_summary` config values
0.1.0
-----
README.md
+38 -12
View File
@@ -1,6 +1,6 @@
# Sphinx llms-full.txt Extension
# Sphinx llms.txt generator
A Sphinx extension that creates a single combined documentation `llms-full.txt` file, written in reStructuredText.
A Sphinx extension that generates a summary `llms.txt` file, written in Markdown, and a single combined documentation `llms-full.txt` file, written in reStructuredText.
## Installation
@@ -20,23 +20,36 @@ extensions = [
## Configuration Options
### `llms_txt_filename`
### `llms_txt_full_file`
- **Type**: boolean
- **Default**: `'True`
- **Description**: Whether to write the single output file
### `llms_txt_full_filename`
- **Type**: string
- **Default**: `'llms-full.txt'`
- **Description**: Name of the output file
- **Description**: Name of the single output file
### `llms_txt_verbose`
- **Type**: boolean
- **Default**: `False`
- **Description**: Whether to include a summary in the build output
### `llms_txt_max_lines`
### `llms_txt_full_max_size`
- **Type**: integer or `None`
- **Default**: `None` (no limit)
- **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes.
- **Description**: Sets a maximum line count for `llms_txt_full_filename`.
If exceeded, the file is skipped and a warning is shown, but the build still completes.
### `llms_txt_file`
- **Type**: boolean
- **Default**: `True`
- **Description**: Whether to write the summary information file
### `llms_txt_filename`
- **Type**: string
- **Default**: `llms.txt`
- **Description**: Name of the summary information file
### `llms_txt_directives`
@@ -44,8 +57,21 @@ extensions = [
- **Default**: `[]` (empty list)
- **Description**: List of custom directive names to process for path resolution.
### `llms_txt_title`
- **Type**: string or `None`
- **Default**: `None`
- **Description**: Overrides the Sphinx project name as the heading in `llms.txt`.
### `llms_txt_summary`
- **Type**: string or `None`
- **Default**: `None`
- **Description**: Optional, but recommended, summary description for `llms.txt`.
## Features
- Creates `llms.txt` and `llms-full.txt`
- Automatically add content from `include` directives
- Resolves relative paths in directives like `image` and `figure` to use full paths
- Ability to add list of custom directives with `llms_txt_directives`
sphinx_llms_txt/__init__.py
+62 -21
View File
@@ -26,6 +26,7 @@ class LLMSFullManager:
self.env: BuildEnvironment = None
self.srcdir: Optional[str] = None
self.outdir: Optional[str] = None
self.app: Optional[Sphinx] = None
def set_master_doc(self, master_doc: str):
"""Set the master document name."""
@@ -44,6 +45,10 @@ class LLMSFullManager:
"""Set configuration options."""
self.config = config
def set_app(self, app: Sphinx):
"""Set the Sphinx application reference."""
self.app = app
def get_page_order(self) -> List[str]:
"""Get the correct page order from the toctree structure."""
if not self.env or not self.master_doc:
@@ -118,7 +123,7 @@ class LLMSFullManager:
return
# Determine output file name and location
output_filename = self.config.get("llms_txt_filename")
output_filename = self.config.get("llms_txt_full_filename")
output_path = Path(outdir) / output_filename
# Find sources directory
@@ -172,7 +177,7 @@ class LLMSFullManager:
# Add pages in order
added_files = set()
total_line_count = 0
max_lines = self.config.get("llms_txt_max_lines")
max_lines = self.config.get("llms_txt_full_max_size")
abort_due_to_max_lines = False
for docname in page_order:
@@ -212,7 +217,7 @@ class LLMSFullManager:
total_line_count += line_count
# Check if line limit was exceeded before creating the file
max_lines = self.config.get("llms_txt_max_lines")
max_lines = self.config.get("llms_txt_full_max_size")
if abort_due_to_max_lines or (
max_lines is not None and total_line_count > max_lines
):
@@ -223,8 +228,8 @@ class LLMSFullManager:
)
# Log summary information if requested
if self.config.get("llms_txt_verbose"):
self._log_summary_info(page_order, total_line_count)
if self.config.get("llms_txt_file"):
self._write_verbose_info_to_file(page_order, total_line_count)
return
@@ -239,8 +244,8 @@ class LLMSFullManager:
)
# Log summary information if requested
if self.config.get("llms_txt_verbose"):
self._log_summary_info(page_order, total_line_count)
if self.config.get("llms_txt_file"):
self._write_verbose_info_to_file(page_order, total_line_count)
except Exception as e:
logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}")
@@ -478,18 +483,45 @@ class LLMSFullManager:
processed_content = include_pattern.sub(replace_include, content)
return processed_content
def _log_summary_info(self, page_order: List[str], total_line_count: int = 0):
"""Log summary information to the logger."""
logger.info("")
logger.info("llms-txt Summary")
logger.info("================")
logger.info(f"Total pages: {len(page_order)}")
def _write_verbose_info_to_file(
self, page_order: List[str], total_line_count: int = 0
):
"""Write summary information to the llms.txt file."""
if not self.outdir:
logger.warning(
"sphinx-llms-txt: Cannot write verbose info to file: outdir not set"
)
return
logger.info(f"Configuration: {self.config}")
logger.info("Page order:")
output_path = Path(self.outdir) / self.config.get("llms_txt_filename")
try:
with open(output_path, "w", encoding="utf-8") as f:
project_name = "llms-txt Summary"
# First priority: use title from config if available
if self.config.get("llms_txt_title"):
project_name = self.config.get("llms_txt_title")
# Second priority: use project name from Sphinx app if available
elif (
self.app
and hasattr(self.app, "config")
and hasattr(self.app.config, "project")
):
project_name = self.app.config.project
f.write(f"# {project_name}\n\n")
# Add description if available
description = self.config.get("llms_txt_summary", "")
if description:
f.write(f"> {description}\n\n")
f.write("## Docs\n\n")
for i, docname in enumerate(page_order, 1):
title = self.page_titles.get(docname, docname)
logger.info(f"{i:3d}. {docname} - {title}")
f.write(f"- [{title}](/{docname}.html)\n")
logger.info(f"sphinx-llms-txt: created {output_path}")
except Exception as e:
logger.error(f"sphinx-llms-txt: Error writing verbose info to file: {e}")
# Global manager instance
@@ -517,12 +549,17 @@ def build_finished(app: Sphinx, exception):
# Set the environment and master doc in the manager
_manager.set_env(app.env)
_manager.set_master_doc(app.config.master_doc)
_manager.set_app(app)
# Set up configuration
config = {
"llms_txt_file": app.config.llms_txt_file,
"llms_txt_filename": app.config.llms_txt_filename,
"llms_txt_verbose": app.config.llms_txt_verbose,
"llms_txt_max_lines": app.config.llms_txt_max_lines,
"llms_txt_title": app.config.llms_txt_title,
"llms_txt_summary": app.config.llms_txt_summary,
"llms_txt_full_file": app.config.llms_txt_full_file,
"llms_txt_full_filename": app.config.llms_txt_full_filename,
"llms_txt_full_max_size": app.config.llms_txt_full_max_size,
"llms_txt_directives": app.config.llms_txt_directives,
"html_baseurl": getattr(app.config, "html_baseurl", ""),
}
@@ -543,10 +580,14 @@ def setup(app: Sphinx) -> Dict[str, Any]:
"""Set up the Sphinx extension."""
# Add configuration options
app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_verbose", False, "env")
app.add_config_value("llms_txt_max_lines", None, "env")
app.add_config_value("llms_txt_file", True, "env")
app.add_config_value("llms_txt_filename", "llms.txt", "env")
app.add_config_value("llms_txt_full_file", True, "env")
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_full_max_size", None, "env")
app.add_config_value("llms_txt_directives", [], "env")
app.add_config_value("llms_txt_title", None, "env")
app.add_config_value("llms_txt_summary", None, "env")
# Connect to Sphinx events
app.connect("doctree-resolved", doctree_resolved)
tests/roots/basic/conf.py
+2 -2
View File
@@ -15,8 +15,8 @@ html_theme = "alabaster"
html_static_path = ["_static"]
# Configuration for sphinx-llms-txt
llms_txt_filename = "test-llms-full.txt"
llms_txt_verbose = True
llms_txt_full_filename = "test-llms-full.txt"
llms_txt_file = True
# Master document
master_doc = "index"
tests/roots/basic/conf_custom.py
+2 -2
View File
@@ -15,8 +15,8 @@ html_theme = "alabaster"
html_static_path = ["_static"]
# Configuration for sphinx-llms-txt
llms_txt_filename = "custom-name.txt"
llms_txt_verbose = True
llms_txt_full_filename = "custom-name.txt"
llms_txt_file = True
# Master document
master_doc = "index"
tests/test_integration.py
+53 -6
View File
@@ -39,6 +39,7 @@ def test_custom_filename(temp_dir, rootdir):
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
print(src_dir)
# Create a copy of the configuration with a different filename
custom_conf = src_dir / "conf_custom.py"
@@ -46,8 +47,8 @@ def test_custom_filename(temp_dir, rootdir):
conf_content = f.read()
conf_content = conf_content.replace(
'llms_txt_filename = "test-llms-full.txt"',
'llms_txt_filename = "custom-name.txt"',
'llms_txt_full_filename = "test-llms-full.txt"',
'llms_txt_full_filename = "custom-name.txt"',
)
with open(custom_conf, "w") as f:
@@ -59,7 +60,7 @@ def test_custom_filename(temp_dir, rootdir):
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={"llms_txt_filename": "custom-name.txt"},
confoverrides={"llms_txt_full_filename": "custom-name.txt"},
)
app.build()
@@ -94,9 +95,8 @@ def test_max_lines_limit(temp_dir, rootdir):
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_filename": "limited.txt",
"llms_txt_max_lines": 10, # Set a small limit to trigger the warning
"llms_txt_verbose": True,
"llms_txt_full_filename": "limited.txt",
"llms_txt_full_max_size": 10, # Set a small limit to trigger the warning
},
)
@@ -115,3 +115,50 @@ def test_max_lines_limit(temp_dir, rootdir):
# Safe unlink
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
def test_title_override(temp_dir, rootdir):
"""Test that the title override works correctly."""
from sphinx.testing.util import SphinxTestApp
src_dir = rootdir / "basic"
# Custom title to override the default project name
custom_title = "Custom Title Override"
# Create a new test app with the title override
app = SphinxTestApp(
srcdir=src_dir,
builddir=temp_dir,
buildername="html",
freshenv=True,
confoverrides={
"llms_txt_title": custom_title,
},
)
app.build()
# Check if the summary file was created
summary_file = Path(app.outdir) / "llms.txt"
assert summary_file.exists(), f"Summary file {summary_file} does not exist"
# Read the content of the summary file
content = summary_file.read_text()
# Check that the custom title was used
assert (
f"# {custom_title}" in content
), f"Custom title '{custom_title}' not found in summary file"
# Ensure the default project name was NOT used
assert (
"# Test Project" not in content
), "Default project name was used instead of custom title"
# Custom cleanup to avoid missing_ok issue
sys.path[:] = app._saved_path
_clean_up_global_state()
# Safe unlink
if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
app.docutils_conf_path.unlink()
tests/test_llms_txt.py
+47 -3
View File
@@ -58,9 +58,9 @@ def test_set_config():
"""Test setting configuration."""
manager = LLMSFullManager()
config = {
"llms_txt_filename": "custom.txt",
"llms_txt_verbose": True,
"llms_txt_max_lines": 1000,
"llms_txt_full_filename": "custom.txt",
"llms_txt_file": True,
"llms_txt_full_max_size": 1000,
}
manager.set_config(config)
assert manager.config == config
@@ -175,3 +175,47 @@ def test_process_includes_with_relative_paths(tmp_path):
" directory.\nLine after include."
)
assert processed_content == expected_content
def test_write_verbose_info_to_file(tmp_path):
"""Test writing verbose info to a file."""
# Create a manager
manager = LLMSFullManager()
# Set up a build directory
build_dir = tmp_path / "build"
build_dir.mkdir()
# Set the outdir on the manager
manager.outdir = str(build_dir)
# Set configuration with verbose_file enabled
config = {
"llms_txt_file": True,
"llms_txt_full_max_size": 1000,
"llms_txt_filename": "llms.txt",
}
manager.set_config(config)
# Add some page titles
manager.update_page_title("index", "Home Page")
manager.update_page_title("about", "About Us")
# Create a page order
page_order = ["index", "about"]
# Call the method to write verbose info to file
manager._write_verbose_info_to_file(page_order, 500)
# Check that the file was created
verbose_file = build_dir / "llms.txt"
assert verbose_file.exists()
# Read the file content
with open(verbose_file, "r", encoding="utf-8") as f:
content = f.read()
# Check that the content contains expected information
assert "## Docs" in content
assert "- [Home Page](/index.html)" in content
assert "- [About Us](/about.html)" in content