From 5bfbb0168f11a89728c075e96db03c743a316348 Mon Sep 17 00:00:00 2001 From: Jared Dillard Date: Mon, 27 Oct 2025 17:41:04 -0700 Subject: [PATCH] Support customizable URI templates in llms.txt (#48) --- CHANGELOG.rst | 6 + docs/source/advanced-configuration.rst | 32 +++++ docs/source/conf.py | 3 +- docs/source/configuration-values.rst | 9 ++ sphinx_llms_txt/__init__.py | 4 +- sphinx_llms_txt/manager.py | 8 +- sphinx_llms_txt/writer.py | 65 ++++++++- tests/test_llms_txt.py | 1 + tests/test_uri_template.py | 175 +++++++++++++++++++++++++ 9 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 tests/test_uri_template.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bf5a9de..49a73c3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +0.7.0 +----- + +- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`. + `#48 `_ + 0.6.0 ----- diff --git a/docs/source/advanced-configuration.rst b/docs/source/advanced-configuration.rst index d457944..1668cbf 100644 --- a/docs/source/advanced-configuration.rst +++ b/docs/source/advanced-configuration.rst @@ -261,6 +261,37 @@ If you want to include absolute URLs for resources in your documentation, you ca When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files. +.. _customizing_uri_links: + +Customizing URI Links in llms.txt +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available. +You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`: + +.. code-block:: python + + # Default: Link to source files, if _sources exists + llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}" + + # Default: Link to HTML pages instead, if _sources doesn't exist + llms_txt_uri_template = "{base_url}{docname}.html" + + # Manual: Link to a custom markdown build + llms_txt_uri_template = "{base_url}{docname}.md" + +.. _available_template_variables: + +Available Template Variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Your URI template can use the following variables: + +- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash) +- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``) +- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists +- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``) + .. _integration_examples: Integration Examples @@ -285,6 +316,7 @@ Here's a complete example showing multiple :doc:`configuration-values`: This is a comprehensive documentation set for our project. It includes API references, usage examples, and tutorials. """ + llms_txt_uri_template = "{base_url}{docname}.md" # Path handling html_baseurl = "https://docs.example.com/" diff --git a/docs/source/conf.py b/docs/source/conf.py index 94a00e9..e3f3f76 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,6 +15,7 @@ import subprocess project = "sphinx-llms-txt" copyright = "Jared Dillard" author = "Jared Dillard" +llms_txt_uri_template = "{base_url}{docname}.md" llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"] llms_txt_summary = """ A Sphinx extension that generates a summary llms.txt file,written in Markdown, @@ -88,7 +89,7 @@ html_theme_options = { "source_directory": "docs/source/", } -html_baseurl = "https://sphinx-llms-txt.readthedocs.org/" +html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/" # -- Options for HTMLHelp output --------------------------------------------- diff --git a/docs/source/configuration-values.rst b/docs/source/configuration-values.rst index 0eef2b5..bca1903 100644 --- a/docs/source/configuration-values.rst +++ b/docs/source/configuration-values.rst @@ -58,6 +58,15 @@ Project Configuration Values .. versionadded:: 0.2.0 +.. confval:: llms_txt_uri_template + + - **Type**: string or ``None`` + - **Default**: ``None`` + - **Description**: Template string for generating URIs in ``llms.txt``. + See :ref:`customizing_uri_links`. + + .. versionadded:: 0.7.0 + .. confval:: llms_txt_directives - **Type**: list of strings diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index 8a14a20..b627682 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -21,7 +21,7 @@ from .manager import LLMSFullManager from .processor import DocumentProcessor from .writer import FileWriter -__version__ = "0.6.0" +__version__ = "0.7.0" # Export classes needed by tests __all__ = [ @@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception): config = { "llms_txt_file": app.config.llms_txt_file, "llms_txt_filename": app.config.llms_txt_filename, + "llms_txt_uri_template": app.config.llms_txt_uri_template, "llms_txt_title": app.config.llms_txt_title, "llms_txt_summary": summary, "llms_txt_full_file": app.config.llms_txt_full_file, @@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_config_value("llms_txt_file", True, "env") app.add_config_value("llms_txt_filename", "llms.txt", "env") + app.add_config_value("llms_txt_uri_template", None, "env") app.add_config_value("llms_txt_full_file", True, "env") app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env") app.add_config_value("llms_txt_full_max_size", None, "env") diff --git a/sphinx_llms_txt/manager.py b/sphinx_llms_txt/manager.py index bd46a7d..3e970c5 100644 --- a/sphinx_llms_txt/manager.py +++ b/sphinx_llms_txt/manager.py @@ -223,6 +223,7 @@ class LLMSFullManager: filtered_page_order, self.collector.page_titles, 0, # No line count since no llms-full.txt + sources_dir, ) # Only warn if user explicitly wants llms-full.txt @@ -523,6 +524,7 @@ class LLMSFullManager: filtered_page_order, self.collector.page_titles, total_line_count, + sources_dir, ) return elif action == "note": @@ -536,6 +538,7 @@ class LLMSFullManager: filtered_page_order, self.collector.page_titles, total_line_count, + sources_dir, ) return elif action == "keep": @@ -554,7 +557,10 @@ class LLMSFullManager: if success and self.config.get("llms_txt_file"): filtered_page_order = self._filter_ignored_pages(page_order) self.writer.write_verbose_info_to_file( - filtered_page_order, self.collector.page_titles, total_line_count + filtered_page_order, + self.collector.page_titles, + total_line_count, + sources_dir, ) def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]: diff --git a/sphinx_llms_txt/writer.py b/sphinx_llms_txt/writer.py index e5f873b..44b9279 100644 --- a/sphinx_llms_txt/writer.py +++ b/sphinx_llms_txt/writer.py @@ -19,6 +19,42 @@ class FileWriter: self.outdir = outdir self.app = app + def _resolve_uri_template(self, sources_dir: Path = None) -> str: + """Resolve which URI template to use based on configuration and sources_dir. + + Args: + sources_dir: Path to _sources directory (None if not found) + + Returns: + The template string to use for generating URIs + """ + # If custom template exists + custom_template = self.config.get("llms_txt_uri_template") + + if custom_template: + # Validate user's template by checking for valid variable names + try: + # Try formatting with test valid values to validate syntax + test_values = { + "base_url": "http://example.com/", + "docname": "test", + "suffix": ".rst", + "sourcelink_suffix": ".txt", + } + custom_template.format(**test_values) + return custom_template + except (KeyError, ValueError) as e: + logger.warning( + f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. " + f"Falling back to default." + ) + + # Else, use one of the default templates + if sources_dir: + return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}" + else: + return "{base_url}{docname}.html" + def write_combined_file( self, content_parts: List[str], output_path: Path, total_line_count: int ) -> bool: @@ -50,6 +86,7 @@ class FileWriter: page_order: Union[List[str], List[Tuple[str, str]]], page_titles: Dict[str, str], total_line_count: int = 0, + sources_dir: Path = None, ) -> bool: """Write summary information to the llms.txt file. @@ -57,6 +94,7 @@ class FileWriter: page_order: Ordered list of document names or (docname, suffix) tuples page_titles: Dictionary mapping docnames to titles total_line_count: Total number of lines in the combined content + sources_dir: Path to _sources directory (None if not found) Returns: True if successful, False otherwise @@ -102,14 +140,37 @@ class FileWriter: if not base_url.endswith("/"): base_url += "/" + # Get sourcelink suffix from Sphinx config + sourcelink_suffix = "" + if self.app and hasattr(self.app.config, "html_sourcelink_suffix"): + sourcelink_suffix = self.app.config.html_sourcelink_suffix + # Handle empty string case specially + if sourcelink_suffix == "": + sourcelink_suffix = "" # Keep it empty + elif not sourcelink_suffix.startswith("."): + sourcelink_suffix = "." + sourcelink_suffix + + # Resolve which template to use + uri_template = self._resolve_uri_template(sources_dir) + for item in page_order: # Handle both old format (str) and new format (tuple) if isinstance(item, tuple): - docname, _ = item + docname, suffix = item else: docname = item + suffix = None + title = page_titles.get(docname, docname) - f.write(f"- [{title}]({base_url}{docname}.html)\n") + + uri = uri_template.format( + base_url=base_url, + docname=docname, + suffix=suffix or "", + sourcelink_suffix=sourcelink_suffix, + ) + + f.write(f"- [{title}]({uri})\n") logger.info(f"sphinx-llms-txt: created {output_path}") return True diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index 3be6fb3..412da2c 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -794,6 +794,7 @@ def test_summary_default_uses_first_paragraph(): llms_txt_summary = None # Not configured llms_txt_file = True llms_txt_filename = "llms.txt" + llms_txt_uri_template = None llms_txt_title = None llms_txt_full_file = True llms_txt_full_filename = "llms-full.txt" diff --git a/tests/test_uri_template.py b/tests/test_uri_template.py new file mode 100644 index 0000000..f5ad243 --- /dev/null +++ b/tests/test_uri_template.py @@ -0,0 +1,175 @@ +"""Test URI template functionality for llms.txt links.""" + +from sphinx_llms_txt import FileWriter + + +def test_uri_template_with_sources_dir(tmp_path): + """Test that default template uses _sources links when sources_dir exists.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + + # Create _sources directory to simulate its existence + sources_dir = build_dir / "_sources" + sources_dir.mkdir() + + # Mock app with html_sourcelink_suffix + class MockApp: + class Config: + html_sourcelink_suffix = ".txt" + + config = Config() + + config = { + "llms_txt_file": True, + "llms_txt_filename": "llms.txt", + "llms_txt_uri_template": ( + "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}" + ), + "html_baseurl": "https://example.com", + } + writer = FileWriter(config, str(build_dir), MockApp()) + + page_titles = { + "index": "Home Page", + "about": "About Us", + } + + # Page order with suffixes (simulating _sources files exist) + page_order = [("index", ".rst"), ("about", ".md")] + + writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir) + + # Check that the file was created + verbose_file = build_dir / "llms.txt" + assert verbose_file.exists() + + # Read the file content + with open(verbose_file, "r", encoding="utf-8") as f: + content = f.read() + + # Should link to _sources files + assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content + assert "- [About Us](https://example.com/_sources/about.md.txt)" in content + + +def test_uri_template_without_sources_dir(tmp_path): + """ + Test that HTML template is used when sources_dir doesn't exist and no custom + template. + """ + build_dir = tmp_path / "build" + build_dir.mkdir() + + config = { + "llms_txt_file": True, + "llms_txt_filename": "llms.txt", + # No custom template set + "html_baseurl": "https://example.com", + } + writer = FileWriter(config, str(build_dir)) + + page_titles = { + "index": "Home Page", + "about": "About Us", + } + + # Page order without suffixes (simulating no _sources) + page_order = [("index", None), ("about", None)] + + # Pass None for sources_dir to simulate it doesn't exist + writer.write_verbose_info_to_file(page_order, page_titles, 0, None) + + # Check that the file was created + verbose_file = build_dir / "llms.txt" + assert verbose_file.exists() + + # Read the file content + with open(verbose_file, "r", encoding="utf-8") as f: + content = f.read() + + # Should fallback to HTML links + assert "- [Home Page](https://example.com/index.html)" in content + assert "- [About Us](https://example.com/about.html)" in content + + +def test_uri_template_custom(tmp_path): + """Test that custom URI template works correctly.""" + build_dir = tmp_path / "build" + build_dir.mkdir() + + sources_dir = build_dir / "_sources" + sources_dir.mkdir() + + # Mock app with html_sourcelink_suffix + class MockApp: + class Config: + html_sourcelink_suffix = ".txt" + + config = Config() + + # Custom template that uses different path + config = { + "llms_txt_file": True, + "llms_txt_filename": "llms.txt", + "llms_txt_uri_template": "{base_url}raw/{docname}{suffix}", + "html_baseurl": "https://example.com/", + } + writer = FileWriter(config, str(build_dir), MockApp()) + + page_titles = { + "index": "Home Page", + } + + page_order = [("index", ".rst")] + + writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir) + + verbose_file = build_dir / "llms.txt" + with open(verbose_file, "r", encoding="utf-8") as f: + content = f.read() + + # Should use custom template + assert "- [Home Page](https://example.com/raw/index.rst)" in content + + +def test_uri_template_invalid_fallback(tmp_path): + """ + Test that invalid template falls back to default sources template when + sources_dir exists. + """ + build_dir = tmp_path / "build" + build_dir.mkdir() + + sources_dir = build_dir / "_sources" + sources_dir.mkdir() + + # Mock app with html_sourcelink_suffix + class MockApp: + class Config: + html_sourcelink_suffix = ".txt" + + config = Config() + + # Invalid template with typo in variable name + config = { + "llms_txt_file": True, + "llms_txt_filename": "llms.txt", + "llms_txt_uri_template": "{base_urll}/{docname}", + "html_baseurl": "https://example.com", + } + writer = FileWriter(config, str(build_dir), MockApp()) + + page_titles = { + "index": "Home Page", + } + + page_order = [("index", ".rst")] + + writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir) + + verbose_file = build_dir / "llms.txt" + with open(verbose_file, "r", encoding="utf-8") as f: + content = f.read() + + # Should fallback to default sources template + assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content