Merge branch 'main' into feature/singlerst-builder

.readthedocs.yml

@@ -1,15 +1,17 @@
 version: 2
 
 build:
-  os: "ubuntu-20.04"
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/source/conf.py
-
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - method: pip
-      path: .
+    python: "3.13"
+  commands:
+    - pip install cmake
+    - pip install -r docs/requirements.txt
+    - pip install -e .
+    - cmake --workflow --preset documentation-workflow
+    # Copy built documentation to Read the Docs output directory
+    - mkdir -p $READTHEDOCS_OUTPUT/html
+    - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/rst/* $READTHEDOCS_OUTPUT/html/
+    - cp build/singlerst/index.rst $READTHEDOCS_OUTPUT/html/llms-full.txt

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+0.7.0
+-----
+
+- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`.
+  `#48 <https://github.com/jdillard/sphinx-llms-txt/pull/48>`_
+
 0.6.0
 -----
 

CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 3.15)
+project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
+
+# Add CMake module path
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+# Add documentation
+add_subdirectory(docs)

CMakePresets.json

@@ -0,0 +1,59 @@
+{
+  "version": 6,
+  "configurePresets": [
+    {
+      "name": "documentation",
+      "displayName": "Documentation Build",
+      "description": "Configure project with documentation environment setup",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "html",
+      "displayName": "Build HTML Documentation",
+      "configurePreset": "documentation",
+      "targets": ["html"]
+    },
+    {
+      "name": "markdown",
+      "displayName": "Build Markdown Documentation",
+      "configurePreset": "documentation",
+      "targets": ["markdown"]
+    },
+    {
+      "name": "rst",
+      "displayName": "Build reStructuredText Documentation",
+      "configurePreset": "documentation",
+      "targets": ["rst"]
+    },
+    {
+      "name": "singlerst",
+      "displayName": "Build Single reStructuredText Documentation",
+      "configurePreset": "documentation",
+      "targets": ["singlerst"]
+    },
+    {
+      "name": "docs-parallel",
+      "displayName": "Build HTML and Markdown in parallel",
+      "configurePreset": "documentation",
+      "targets": ["html", "markdown", "rst", "singlerst"]
+    }
+  ],
+  "workflowPresets": [
+    {
+      "name": "documentation-workflow",
+      "displayName": "Documentation Build Workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "documentation"
+        },
+        {
+          "type": "build",
+          "name": "docs-parallel"
+        }
+      ]
+    }
+  ]
+}

cmake/SphinxUtils.cmake

@@ -0,0 +1,28 @@
+# Sphinx related utilities
+
+set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
+
+# Function to find Sphinx in the system
+function(setup_sphinx_environment)
+    # Find sphinx-build executable in system
+    find_program(SPHINX_EXECUTABLE
+        NAMES sphinx-build
+        DOC "Sphinx documentation generator"
+    )
+
+    if(NOT SPHINX_EXECUTABLE)
+        message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
+    endif()
+
+    # Export to parent scope
+    set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
+endfunction()
+
+# Function to add a Sphinx builder target
+function(add_sphinx_builder builder_name)
+    add_custom_target(${builder_name}
+        COMMAND ${SPHINX_EXECUTABLE} -b ${builder_name} ${SPHINX_SOURCE} ${SPHINX_BUILD}/${builder_name}
+        VERBATIM
+    )
+endfunction()

docs/CMakeLists.txt

@@ -0,0 +1,8 @@
+include(SphinxUtils)
+
+setup_sphinx_environment()
+
+add_sphinx_builder(html)
+add_sphinx_builder(markdown)
+add_sphinx_builder(rst)
+add_sphinx_builder(singlerst)

docs/requirements.txt

@@ -3,4 +3,7 @@ esbonio
 sphinx-contributors
 sphinx
 sphinx-llms-txt
+sphinx-inline-tabs
 sphinxext-opengraph
+sphinx-markdown-builder
+sphinxcontrib-restbuilder @ git+https://github.com/jdillard/restbuilder.git@feature/singlerst-builder

docs/source/advanced-configuration.rst

@@ -261,6 +261,42 @@ If you want to include absolute URLs for resources in your documentation, you ca
 
 When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
 
+.. _customizing_uri_links:
+
+Customizing URI Links in llms.txt
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available.
+You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`:
+
+.. code-block:: python
+
+   # Default: Link to source files, if _sources exists
+   llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+
+   # Default: Link to HTML pages instead, if _sources doesn't exist
+   llms_txt_uri_template = "{base_url}{docname}.html"
+
+   # Manual: Link to a custom markdown build
+   llms_txt_uri_template = "{base_url}{docname}.md"
+
+.. _available_template_variables:
+
+Available Template Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Your URI template can use the following variables:
+
+- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash)
+- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``)
+- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
+- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
+
+.. tip::
+   Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
+   See this package's `CMake setup <https://github.com/jdillard/sphinx-llms-txt>`_ for an example of building both HTML and Markdown and/or reStructuredText in parallel.
+   Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
+
 .. _integration_examples:
 
 Integration Examples
@@ -285,6 +321,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
    This is a comprehensive documentation set for our project.
    It includes API references, usage examples, and tutorials.
    """
+   llms_txt_uri_template = "{base_url}{docname}.md"
 
    # Path handling
    html_baseurl = "https://docs.example.com/"

docs/source/conf.py

@@ -15,12 +15,18 @@ import subprocess
 project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
+
+llms_txt_uri_template = "{base_url}{docname}.md"
+llms_txt_full_file = False
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
 A Sphinx extension that generates a summary llms.txt file,written in Markdown,
 and a single combined documentation llms-full.txt file, written in reStructuredText.
 """
 
+# This doesn't seem to be supported
+# rst_file_suffix = ".html.rst"
+
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
     GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -49,6 +55,8 @@ extensions = [
     "sphinx.ext.intersphinx",
     "sphinx_contributors",
     "sphinx_llms_txt",
+    "sphinxcontrib.restbuilder",
+    "sphinx_inline_tabs",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -88,7 +96,7 @@ html_theme_options = {
     "source_directory": "docs/source/",
 }
 
-html_baseurl = "https://sphinx-llms-txt.readthedocs.org/"
+html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/"
 
 
 # -- Options for HTMLHelp output ---------------------------------------------

docs/source/configuration-values.rst

@@ -58,6 +58,15 @@ Project Configuration Values
 
    .. versionadded:: 0.2.0
 
+.. confval:: llms_txt_uri_template
+
+   - **Type**: string or ``None``
+   - **Default**: ``None``
+   - **Description**: Template string for generating URIs in ``llms.txt``.
+     See :ref:`customizing_uri_links`.
+
+   .. versionadded:: 0.7.0
+
 .. confval:: llms_txt_directives
 
    - **Type**: list of strings

docs/source/getting-started.rst

@@ -4,17 +4,19 @@ Getting Started
 Installation
 ------------
 
-Directly install via ``pip`` by using:
+Directly install by using:
 
-.. code-block:: bash
+.. tab:: via pip
 
-    pip install sphinx-llms-txt
+   .. code-block:: bash
 
-Or with ``conda`` via ``conda-forge``:
+      pip install sphinx-llms-txt
 
-.. code::
+.. tab:: via conda:
 
-   conda install -c conda-forge sphinx-llms-txt
+   .. code-block:: bash
+
+      conda install -c conda-forge sphinx-llms-txt
 
 Usage
 -----

pyproject.toml

@@ -48,6 +48,8 @@ test = [
     "pytest>=7.0.0",
 ]
 
+[tool.setuptools]
+  packages = ["sphinx_llms_txt"]
 
 [tool.setuptools.dynamic]
 version = {attr = "sphinx_llms_txt.__version__"}
@@ -88,4 +90,4 @@ filterwarnings = [
     "ignore::UserWarning",
     "ignore::DeprecationWarning",
     "ignore::PendingDeprecationWarning",
-]
+]

sphinx_llms_txt/__init__.py

@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 # Export classes needed by tests
 __all__ = [
@@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception):
         config = {
             "llms_txt_file": app.config.llms_txt_file,
             "llms_txt_filename": app.config.llms_txt_filename,
+            "llms_txt_uri_template": app.config.llms_txt_uri_template,
             "llms_txt_title": app.config.llms_txt_title,
             "llms_txt_summary": summary,
             "llms_txt_full_file": app.config.llms_txt_full_file,
@@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
 
     app.add_config_value("llms_txt_file", True, "env")
     app.add_config_value("llms_txt_filename", "llms.txt", "env")
+    app.add_config_value("llms_txt_uri_template", None, "env")
     app.add_config_value("llms_txt_full_file", True, "env")
     app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
     app.add_config_value("llms_txt_full_max_size", None, "env")

sphinx_llms_txt/manager.py

@@ -223,6 +223,7 @@ class LLMSFullManager:
                     filtered_page_order,
                     self.collector.page_titles,
                     0,  # No line count since no llms-full.txt
+                    sources_dir,
                 )
 
             # Only warn if user explicitly wants llms-full.txt
@@ -523,6 +524,7 @@ class LLMSFullManager:
                         filtered_page_order,
                         self.collector.page_titles,
                         total_line_count,
+                        sources_dir,
                     )
                 return
             elif action == "note":
@@ -536,6 +538,7 @@ class LLMSFullManager:
                         filtered_page_order,
                         self.collector.page_titles,
                         total_line_count,
+                        sources_dir,
                     )
                 return
             elif action == "keep":
@@ -554,7 +557,10 @@ class LLMSFullManager:
         if success and self.config.get("llms_txt_file"):
             filtered_page_order = self._filter_ignored_pages(page_order)
             self.writer.write_verbose_info_to_file(
-                filtered_page_order, self.collector.page_titles, total_line_count
+                filtered_page_order,
+                self.collector.page_titles,
+                total_line_count,
+                sources_dir,
             )
 
     def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:

sphinx_llms_txt/writer.py

@@ -19,6 +19,42 @@ class FileWriter:
         self.outdir = outdir
         self.app = app
 
+    def _resolve_uri_template(self, sources_dir: Path = None) -> str:
+        """Resolve which URI template to use based on configuration and sources_dir.
+
+        Args:
+            sources_dir: Path to _sources directory (None if not found)
+
+        Returns:
+            The template string to use for generating URIs
+        """
+        # If custom template exists
+        custom_template = self.config.get("llms_txt_uri_template")
+
+        if custom_template:
+            # Validate user's template by checking for valid variable names
+            try:
+                # Try formatting with test valid values to validate syntax
+                test_values = {
+                    "base_url": "http://example.com/",
+                    "docname": "test",
+                    "suffix": ".rst",
+                    "sourcelink_suffix": ".txt",
+                }
+                custom_template.format(**test_values)
+                return custom_template
+            except (KeyError, ValueError) as e:
+                logger.warning(
+                    f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. "
+                    f"Falling back to default."
+                )
+
+        # Else, use one of the default templates
+        if sources_dir:
+            return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+        else:
+            return "{base_url}{docname}.html"
+
     def write_combined_file(
         self, content_parts: List[str], output_path: Path, total_line_count: int
     ) -> bool:
@@ -50,6 +86,7 @@ class FileWriter:
         page_order: Union[List[str], List[Tuple[str, str]]],
         page_titles: Dict[str, str],
         total_line_count: int = 0,
+        sources_dir: Path = None,
     ) -> bool:
         """Write summary information to the llms.txt file.
 
@@ -57,6 +94,7 @@ class FileWriter:
             page_order: Ordered list of document names or (docname, suffix) tuples
             page_titles: Dictionary mapping docnames to titles
             total_line_count: Total number of lines in the combined content
+            sources_dir: Path to _sources directory (None if not found)
 
         Returns:
             True if successful, False otherwise
@@ -102,14 +140,37 @@ class FileWriter:
                 if not base_url.endswith("/"):
                     base_url += "/"
 
+                # Get sourcelink suffix from Sphinx config
+                sourcelink_suffix = ""
+                if self.app and hasattr(self.app.config, "html_sourcelink_suffix"):
+                    sourcelink_suffix = self.app.config.html_sourcelink_suffix
+                    # Handle empty string case specially
+                    if sourcelink_suffix == "":
+                        sourcelink_suffix = ""  # Keep it empty
+                    elif not sourcelink_suffix.startswith("."):
+                        sourcelink_suffix = "." + sourcelink_suffix
+
+                # Resolve which template to use
+                uri_template = self._resolve_uri_template(sources_dir)
+
                 for item in page_order:
                     # Handle both old format (str) and new format (tuple)
                     if isinstance(item, tuple):
-                        docname, _ = item
+                        docname, suffix = item
                     else:
                         docname = item
+                        suffix = None
+
                     title = page_titles.get(docname, docname)
-                    f.write(f"- [{title}]({base_url}{docname}.html)\n")
+
+                    uri = uri_template.format(
+                        base_url=base_url,
+                        docname=docname,
+                        suffix=suffix or "",
+                        sourcelink_suffix=sourcelink_suffix,
+                    )
+
+                    f.write(f"- [{title}]({uri})\n")
 
             logger.info(f"sphinx-llms-txt: created {output_path}")
             return True

tests/test_llms_txt.py

@@ -794,6 +794,7 @@ def test_summary_default_uses_first_paragraph():
             llms_txt_summary = None  # Not configured
             llms_txt_file = True
             llms_txt_filename = "llms.txt"
+            llms_txt_uri_template = None
             llms_txt_title = None
             llms_txt_full_file = True
             llms_txt_full_filename = "llms-full.txt"

tests/test_uri_template.py

@@ -0,0 +1,175 @@
+"""Test URI template functionality for llms.txt links."""
+
+from sphinx_llms_txt import FileWriter
+
+
+def test_uri_template_with_sources_dir(tmp_path):
+    """Test that default template uses _sources links when sources_dir exists."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    # Create _sources directory to simulate its existence
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": (
+            "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+        ),
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+        "about": "About Us",
+    }
+
+    # Page order with suffixes (simulating _sources files exist)
+    page_order = [("index", ".rst"), ("about", ".md")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    # Check that the file was created
+    verbose_file = build_dir / "llms.txt"
+    assert verbose_file.exists()
+
+    # Read the file content
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should link to _sources files
+    assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
+    assert "- [About Us](https://example.com/_sources/about.md.txt)" in content
+
+
+def test_uri_template_without_sources_dir(tmp_path):
+    """
+    Test that HTML template is used when sources_dir doesn't exist and no custom
+    template.
+    """
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        # No custom template set
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir))
+
+    page_titles = {
+        "index": "Home Page",
+        "about": "About Us",
+    }
+
+    # Page order without suffixes (simulating no _sources)
+    page_order = [("index", None), ("about", None)]
+
+    # Pass None for sources_dir to simulate it doesn't exist
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, None)
+
+    # Check that the file was created
+    verbose_file = build_dir / "llms.txt"
+    assert verbose_file.exists()
+
+    # Read the file content
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should fallback to HTML links
+    assert "- [Home Page](https://example.com/index.html)" in content
+    assert "- [About Us](https://example.com/about.html)" in content
+
+
+def test_uri_template_custom(tmp_path):
+    """Test that custom URI template works correctly."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    # Custom template that uses different path
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": "{base_url}raw/{docname}{suffix}",
+        "html_baseurl": "https://example.com/",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+    }
+
+    page_order = [("index", ".rst")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    verbose_file = build_dir / "llms.txt"
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should use custom template
+    assert "- [Home Page](https://example.com/raw/index.rst)" in content
+
+
+def test_uri_template_invalid_fallback(tmp_path):
+    """
+    Test that invalid template falls back to default sources template when
+    sources_dir exists.
+    """
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    # Invalid template with typo in variable name
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": "{base_urll}/{docname}",
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+    }
+
+    page_order = [("index", ".rst")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    verbose_file = build_dir / "llms.txt"
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should fallback to default sources template
+    assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content