From 10e2b9a68494ca014ca6964f36387f5917c5b388 Mon Sep 17 00:00:00 2001 From: Jared Dillard Date: Sat, 17 May 2025 20:22:46 -0700 Subject: [PATCH] Add llms.txt config options (#9) --- CHANGELOG.rst | 7 +-- README.md | 50 +++++++++++++----- sphinx_llms_txt/__init__.py | 87 +++++++++++++++++++++++--------- tests/roots/basic/conf.py | 4 +- tests/roots/basic/conf_custom.py | 4 +- tests/test_integration.py | 59 +++++++++++++++++++--- tests/test_llms_txt.py | 50 ++++++++++++++++-- 7 files changed, 210 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e0450c6..4b2c2a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,9 +4,10 @@ Changelog 0.2.0 ----- -- Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size -- Automatically add content from `.. include::` directives -- Add path resolution for certain directives +- Add `llms_txt_full_max_size` configuration option to limit `llms-full.txt` file size +- Automatically add content from `include` directives in `llms-full.txt` +- Add path resolution for a given set of directives in `llms-full.txt` +- Add `llms.txt` file option, with `llms_txt_title` and `llms_txt_summary` config values 0.1.0 ----- diff --git a/README.md b/README.md index 3bdabb7..8243325 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Sphinx llms-full.txt Extension +# Sphinx llms.txt generator -A Sphinx extension that creates a single combined documentation `llms-full.txt` file, written in reStructuredText. +A Sphinx extension that generates a summary `llms.txt` file, written in Markdown, and a single combined documentation `llms-full.txt` file, written in reStructuredText. ## Installation @@ -20,23 +20,36 @@ extensions = [ ## Configuration Options -### `llms_txt_filename` +### `llms_txt_full_file` + +- **Type**: boolean +- **Default**: `'True` +- **Description**: Whether to write the single output file + +### `llms_txt_full_filename` - **Type**: string - **Default**: `'llms-full.txt'` -- **Description**: Name of the output file +- **Description**: Name of the single output file -### `llms_txt_verbose` - -- **Type**: boolean -- **Default**: `False` -- **Description**: Whether to include a summary in the build output - -### `llms_txt_max_lines` +### `llms_txt_full_max_size` - **Type**: integer or `None` - **Default**: `None` (no limit) -- **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes. +- **Description**: Sets a maximum line count for `llms_txt_full_filename`. + If exceeded, the file is skipped and a warning is shown, but the build still completes. + +### `llms_txt_file` + +- **Type**: boolean +- **Default**: `True` +- **Description**: Whether to write the summary information file + +### `llms_txt_filename` + +- **Type**: string +- **Default**: `llms.txt` +- **Description**: Name of the summary information file ### `llms_txt_directives` @@ -44,8 +57,21 @@ extensions = [ - **Default**: `[]` (empty list) - **Description**: List of custom directive names to process for path resolution. +### `llms_txt_title` + +- **Type**: string or `None` +- **Default**: `None` +- **Description**: Overrides the Sphinx project name as the heading in `llms.txt`. + +### `llms_txt_summary` + +- **Type**: string or `None` +- **Default**: `None` +- **Description**: Optional, but recommended, summary description for `llms.txt`. + ## Features +- Creates `llms.txt` and `llms-full.txt` - Automatically add content from `include` directives - Resolves relative paths in directives like `image` and `figure` to use full paths - Ability to add list of custom directives with `llms_txt_directives` diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index e371915..609da83 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -26,6 +26,7 @@ class LLMSFullManager: self.env: BuildEnvironment = None self.srcdir: Optional[str] = None self.outdir: Optional[str] = None + self.app: Optional[Sphinx] = None def set_master_doc(self, master_doc: str): """Set the master document name.""" @@ -44,6 +45,10 @@ class LLMSFullManager: """Set configuration options.""" self.config = config + def set_app(self, app: Sphinx): + """Set the Sphinx application reference.""" + self.app = app + def get_page_order(self) -> List[str]: """Get the correct page order from the toctree structure.""" if not self.env or not self.master_doc: @@ -118,7 +123,7 @@ class LLMSFullManager: return # Determine output file name and location - output_filename = self.config.get("llms_txt_filename") + output_filename = self.config.get("llms_txt_full_filename") output_path = Path(outdir) / output_filename # Find sources directory @@ -172,7 +177,7 @@ class LLMSFullManager: # Add pages in order added_files = set() total_line_count = 0 - max_lines = self.config.get("llms_txt_max_lines") + max_lines = self.config.get("llms_txt_full_max_size") abort_due_to_max_lines = False for docname in page_order: @@ -212,7 +217,7 @@ class LLMSFullManager: total_line_count += line_count # Check if line limit was exceeded before creating the file - max_lines = self.config.get("llms_txt_max_lines") + max_lines = self.config.get("llms_txt_full_max_size") if abort_due_to_max_lines or ( max_lines is not None and total_line_count > max_lines ): @@ -223,8 +228,8 @@ class LLMSFullManager: ) # Log summary information if requested - if self.config.get("llms_txt_verbose"): - self._log_summary_info(page_order, total_line_count) + if self.config.get("llms_txt_file"): + self._write_verbose_info_to_file(page_order, total_line_count) return @@ -239,8 +244,8 @@ class LLMSFullManager: ) # Log summary information if requested - if self.config.get("llms_txt_verbose"): - self._log_summary_info(page_order, total_line_count) + if self.config.get("llms_txt_file"): + self._write_verbose_info_to_file(page_order, total_line_count) except Exception as e: logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}") @@ -478,18 +483,45 @@ class LLMSFullManager: processed_content = include_pattern.sub(replace_include, content) return processed_content - def _log_summary_info(self, page_order: List[str], total_line_count: int = 0): - """Log summary information to the logger.""" - logger.info("") - logger.info("llms-txt Summary") - logger.info("================") - logger.info(f"Total pages: {len(page_order)}") + def _write_verbose_info_to_file( + self, page_order: List[str], total_line_count: int = 0 + ): + """Write summary information to the llms.txt file.""" + if not self.outdir: + logger.warning( + "sphinx-llms-txt: Cannot write verbose info to file: outdir not set" + ) + return - logger.info(f"Configuration: {self.config}") - logger.info("Page order:") - for i, docname in enumerate(page_order, 1): - title = self.page_titles.get(docname, docname) - logger.info(f"{i:3d}. {docname} - {title}") + output_path = Path(self.outdir) / self.config.get("llms_txt_filename") + try: + with open(output_path, "w", encoding="utf-8") as f: + project_name = "llms-txt Summary" + # First priority: use title from config if available + if self.config.get("llms_txt_title"): + project_name = self.config.get("llms_txt_title") + # Second priority: use project name from Sphinx app if available + elif ( + self.app + and hasattr(self.app, "config") + and hasattr(self.app.config, "project") + ): + project_name = self.app.config.project + f.write(f"# {project_name}\n\n") + + # Add description if available + description = self.config.get("llms_txt_summary", "") + if description: + f.write(f"> {description}\n\n") + + f.write("## Docs\n\n") + for i, docname in enumerate(page_order, 1): + title = self.page_titles.get(docname, docname) + f.write(f"- [{title}](/{docname}.html)\n") + + logger.info(f"sphinx-llms-txt: created {output_path}") + except Exception as e: + logger.error(f"sphinx-llms-txt: Error writing verbose info to file: {e}") # Global manager instance @@ -517,12 +549,17 @@ def build_finished(app: Sphinx, exception): # Set the environment and master doc in the manager _manager.set_env(app.env) _manager.set_master_doc(app.config.master_doc) + _manager.set_app(app) # Set up configuration config = { + "llms_txt_file": app.config.llms_txt_file, "llms_txt_filename": app.config.llms_txt_filename, - "llms_txt_verbose": app.config.llms_txt_verbose, - "llms_txt_max_lines": app.config.llms_txt_max_lines, + "llms_txt_title": app.config.llms_txt_title, + "llms_txt_summary": app.config.llms_txt_summary, + "llms_txt_full_file": app.config.llms_txt_full_file, + "llms_txt_full_filename": app.config.llms_txt_full_filename, + "llms_txt_full_max_size": app.config.llms_txt_full_max_size, "llms_txt_directives": app.config.llms_txt_directives, "html_baseurl": getattr(app.config, "html_baseurl", ""), } @@ -543,10 +580,14 @@ def setup(app: Sphinx) -> Dict[str, Any]: """Set up the Sphinx extension.""" # Add configuration options - app.add_config_value("llms_txt_filename", "llms-full.txt", "env") - app.add_config_value("llms_txt_verbose", False, "env") - app.add_config_value("llms_txt_max_lines", None, "env") + app.add_config_value("llms_txt_file", True, "env") + app.add_config_value("llms_txt_filename", "llms.txt", "env") + app.add_config_value("llms_txt_full_file", True, "env") + app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env") + app.add_config_value("llms_txt_full_max_size", None, "env") app.add_config_value("llms_txt_directives", [], "env") + app.add_config_value("llms_txt_title", None, "env") + app.add_config_value("llms_txt_summary", None, "env") # Connect to Sphinx events app.connect("doctree-resolved", doctree_resolved) diff --git a/tests/roots/basic/conf.py b/tests/roots/basic/conf.py index 5c792e5..225901d 100644 --- a/tests/roots/basic/conf.py +++ b/tests/roots/basic/conf.py @@ -15,8 +15,8 @@ html_theme = "alabaster" html_static_path = ["_static"] # Configuration for sphinx-llms-txt -llms_txt_filename = "test-llms-full.txt" -llms_txt_verbose = True +llms_txt_full_filename = "test-llms-full.txt" +llms_txt_file = True # Master document master_doc = "index" diff --git a/tests/roots/basic/conf_custom.py b/tests/roots/basic/conf_custom.py index ffa6c3f..ab3f80c 100644 --- a/tests/roots/basic/conf_custom.py +++ b/tests/roots/basic/conf_custom.py @@ -15,8 +15,8 @@ html_theme = "alabaster" html_static_path = ["_static"] # Configuration for sphinx-llms-txt -llms_txt_filename = "custom-name.txt" -llms_txt_verbose = True +llms_txt_full_filename = "custom-name.txt" +llms_txt_file = True # Master document master_doc = "index" diff --git a/tests/test_integration.py b/tests/test_integration.py index 79abe9e..6a3350d 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -39,6 +39,7 @@ def test_custom_filename(temp_dir, rootdir): from sphinx.testing.util import SphinxTestApp src_dir = rootdir / "basic" + print(src_dir) # Create a copy of the configuration with a different filename custom_conf = src_dir / "conf_custom.py" @@ -46,8 +47,8 @@ def test_custom_filename(temp_dir, rootdir): conf_content = f.read() conf_content = conf_content.replace( - 'llms_txt_filename = "test-llms-full.txt"', - 'llms_txt_filename = "custom-name.txt"', + 'llms_txt_full_filename = "test-llms-full.txt"', + 'llms_txt_full_filename = "custom-name.txt"', ) with open(custom_conf, "w") as f: @@ -59,7 +60,7 @@ def test_custom_filename(temp_dir, rootdir): builddir=temp_dir, buildername="html", freshenv=True, - confoverrides={"llms_txt_filename": "custom-name.txt"}, + confoverrides={"llms_txt_full_filename": "custom-name.txt"}, ) app.build() @@ -94,9 +95,8 @@ def test_max_lines_limit(temp_dir, rootdir): buildername="html", freshenv=True, confoverrides={ - "llms_txt_filename": "limited.txt", - "llms_txt_max_lines": 10, # Set a small limit to trigger the warning - "llms_txt_verbose": True, + "llms_txt_full_filename": "limited.txt", + "llms_txt_full_max_size": 10, # Set a small limit to trigger the warning }, ) @@ -115,3 +115,50 @@ def test_max_lines_limit(temp_dir, rootdir): # Safe unlink if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): app.docutils_conf_path.unlink() + + +def test_title_override(temp_dir, rootdir): + """Test that the title override works correctly.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + # Custom title to override the default project name + custom_title = "Custom Title Override" + + # Create a new test app with the title override + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_title": custom_title, + }, + ) + + app.build() + + # Check if the summary file was created + summary_file = Path(app.outdir) / "llms.txt" + assert summary_file.exists(), f"Summary file {summary_file} does not exist" + + # Read the content of the summary file + content = summary_file.read_text() + + # Check that the custom title was used + assert ( + f"# {custom_title}" in content + ), f"Custom title '{custom_title}' not found in summary file" + # Ensure the default project name was NOT used + assert ( + "# Test Project" not in content + ), "Default project name was used instead of custom title" + + # Custom cleanup to avoid missing_ok issue + sys.path[:] = app._saved_path + _clean_up_global_state() + + # Safe unlink + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index a781612..d2f94ed 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -58,9 +58,9 @@ def test_set_config(): """Test setting configuration.""" manager = LLMSFullManager() config = { - "llms_txt_filename": "custom.txt", - "llms_txt_verbose": True, - "llms_txt_max_lines": 1000, + "llms_txt_full_filename": "custom.txt", + "llms_txt_file": True, + "llms_txt_full_max_size": 1000, } manager.set_config(config) assert manager.config == config @@ -175,3 +175,47 @@ def test_process_includes_with_relative_paths(tmp_path): " directory.\nLine after include." ) assert processed_content == expected_content + + +def test_write_verbose_info_to_file(tmp_path): + """Test writing verbose info to a file.""" + # Create a manager + manager = LLMSFullManager() + + # Set up a build directory + build_dir = tmp_path / "build" + build_dir.mkdir() + + # Set the outdir on the manager + manager.outdir = str(build_dir) + + # Set configuration with verbose_file enabled + config = { + "llms_txt_file": True, + "llms_txt_full_max_size": 1000, + "llms_txt_filename": "llms.txt", + } + manager.set_config(config) + + # Add some page titles + manager.update_page_title("index", "Home Page") + manager.update_page_title("about", "About Us") + + # Create a page order + page_order = ["index", "about"] + + # Call the method to write verbose info to file + manager._write_verbose_info_to_file(page_order, 500) + + # Check that the file was created + verbose_file = build_dir / "llms.txt" + assert verbose_file.exists() + + # Read the file content + with open(verbose_file, "r", encoding="utf-8") as f: + content = f.read() + + # Check that the content contains expected information + assert "## Docs" in content + assert "- [Home Page](/index.html)" in content + assert "- [About Us](/about.html)" in content