Compare commits

..
Author SHA1 Message Date
Jared Dillard 4b21010078 Merge branch 'main' into feature/singlerst-builder 2025-12-02 22:10:15 -08:00
Jared Dillard c23c6d4468 Add tabs 2025-12-02 22:09:14 -08:00
Jared Dillard 72719e881c Merge branch 'main' into feature/singlerst-builder 2025-12-02 16:51:43 -08:00
Jared Dillard db32dc60a7 Improve CMake bits 2025-12-02 16:48:29 -08:00
Jared Dillard dc9f16d454 Add singlerst builder 2025-12-01 13:51:26 -08:00
Jared DillardandGitHub 543efabebb [docs] Add restbuilder builder for single page .rst builds (#53) 2025-12-01 13:34:45 -08:00
Jared Dillard 141e0e29f6 clean up the cmake 2025-10-28 12:26:45 -07:00
Jared DillardandGitHub 1b08b2f362 Add docs on markdown usage (#50) 2025-10-27 20:33:28 -07:00
Jared DillardandGitHub 5bfbb0168f Support customizable URI templates in llms.txt (#48) 2025-10-27 17:41:04 -07:00
Jared DillardandGitHub e2a80faf04 Add CMake support for also building Markdown docs in parallel (#49) 2025-10-26 20:11:09 -07:00
Jared DillardandGitHub b63801bcff Improve _sources directory handling (#47) 2025-10-13 23:53:46 -07:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
c45ebb0369 Bump the all-github-actions group with 2 updates (#45)
Bumps the all-github-actions group with 2 updates: [actions/checkout](https://github.com/actions/checkout) and [actions/setup-python](https://github.com/actions/setup-python).


Updates `actions/checkout` from 4 to 5
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v4...v5)

Updates `actions/setup-python` from 5 to 6
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '5'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
- dependency-name: actions/setup-python
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-10-04 18:59:29 -07:00
Jared DillardandGitHub f5dcd15889 Fix optional sphinx dependency (#44) 2025-09-17 10:51:07 -07:00
Jared DillardandGitHub e64e20133a Remove support for singlehtml (#40) 2025-08-29 15:22:32 -07:00
19 changed files with 623 additions and 45 deletions
+5 -5
View File
@@ -10,9 +10,9 @@ jobs:
pre-commit: pre-commit:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v5
- name: Set up Python 3.10 - name: Set up Python 3.10
uses: actions/setup-python@v5 uses: actions/setup-python@v6
with: with:
python-version: "3.10" python-version: "3.10"
- uses: pre-commit/action@v3.0.1 - uses: pre-commit/action@v3.0.1
@@ -23,17 +23,17 @@ jobs:
python-version: ['3.9', '3.10', '3.11', '3.12'] python-version: ['3.9', '3.10', '3.11', '3.12']
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v5
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5 uses: actions/setup-python@v6
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip python -m pip install --upgrade pip
pip install -e ".[dev]" pip install -e . --group dev
# - name: Run mypy # - name: Run mypy
# run: | # run: |
+13 -11
View File
@@ -1,15 +1,17 @@
version: 2 version: 2
build: build:
os: "ubuntu-20.04" os: ubuntu-24.04
tools: tools:
python: "3.10" python: "3.13"
commands:
sphinx: - pip install cmake
configuration: docs/source/conf.py - pip install -r docs/requirements.txt
- pip install -e .
python: - cmake --workflow --preset documentation-workflow
install: # Copy built documentation to Read the Docs output directory
- requirements: docs/requirements.txt - mkdir -p $READTHEDOCS_OUTPUT/html
- method: pip - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
path: . - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
- cp -r build/rst/* $READTHEDOCS_OUTPUT/html/
- cp build/singlerst/index.rst $READTHEDOCS_OUTPUT/html/llms-full.txt
+24
View File
@@ -1,6 +1,30 @@
Changelog Changelog
========= =========
0.7.0
-----
- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`.
`#48 <https://github.com/jdillard/sphinx-llms-txt/pull/48>`_
0.6.0
-----
- Improve _sources directory handling
`#47 <https://github.com/jdillard/sphinx-llms-txt/pull/47>`_
0.5.3
-----
- Make sphinx a required dependency since there are imports from Sphinx
`#44 <https://github.com/jdillard/sphinx-llms-txt/pull/44>`_
0.5.2
-----
- Remove support for singlehtml
`#40 <https://github.com/jdillard/sphinx-llms-txt/pull/40>`_
0.5.1 0.5.1
----- -----
+8
View File
@@ -0,0 +1,8 @@
cmake_minimum_required(VERSION 3.15)
project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
# Add CMake module path
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
# Add documentation
add_subdirectory(docs)
+59
View File
@@ -0,0 +1,59 @@
{
"version": 6,
"configurePresets": [
{
"name": "documentation",
"displayName": "Documentation Build",
"description": "Configure project with documentation environment setup",
"binaryDir": "${sourceDir}/build"
}
],
"buildPresets": [
{
"name": "html",
"displayName": "Build HTML Documentation",
"configurePreset": "documentation",
"targets": ["html"]
},
{
"name": "markdown",
"displayName": "Build Markdown Documentation",
"configurePreset": "documentation",
"targets": ["markdown"]
},
{
"name": "rst",
"displayName": "Build reStructuredText Documentation",
"configurePreset": "documentation",
"targets": ["rst"]
},
{
"name": "singlerst",
"displayName": "Build Single reStructuredText Documentation",
"configurePreset": "documentation",
"targets": ["singlerst"]
},
{
"name": "docs-parallel",
"displayName": "Build HTML and Markdown in parallel",
"configurePreset": "documentation",
"targets": ["html", "markdown", "rst", "singlerst"]
}
],
"workflowPresets": [
{
"name": "documentation-workflow",
"displayName": "Documentation Build Workflow",
"steps": [
{
"type": "configure",
"name": "documentation"
},
{
"type": "build",
"name": "docs-parallel"
}
]
}
]
}
+28
View File
@@ -0,0 +1,28 @@
# Sphinx related utilities
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
# Function to find Sphinx in the system
function(setup_sphinx_environment)
# Find sphinx-build executable in system
find_program(SPHINX_EXECUTABLE
NAMES sphinx-build
DOC "Sphinx documentation generator"
)
if(NOT SPHINX_EXECUTABLE)
message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
endif()
# Export to parent scope
set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
endfunction()
# Function to add a Sphinx builder target
function(add_sphinx_builder builder_name)
add_custom_target(${builder_name}
COMMAND ${SPHINX_EXECUTABLE} -b ${builder_name} ${SPHINX_SOURCE} ${SPHINX_BUILD}/${builder_name}
VERBATIM
)
endfunction()
+8
View File
@@ -0,0 +1,8 @@
include(SphinxUtils)
setup_sphinx_environment()
add_sphinx_builder(html)
add_sphinx_builder(markdown)
add_sphinx_builder(rst)
add_sphinx_builder(singlerst)
+3
View File
@@ -3,4 +3,7 @@ esbonio
sphinx-contributors sphinx-contributors
sphinx sphinx
sphinx-llms-txt sphinx-llms-txt
sphinx-inline-tabs
sphinxext-opengraph sphinxext-opengraph
sphinx-markdown-builder
sphinxcontrib-restbuilder @ git+https://github.com/jdillard/restbuilder.git@feature/singlerst-builder
+37
View File
@@ -261,6 +261,42 @@ If you want to include absolute URLs for resources in your documentation, you ca
When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files. When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
.. _customizing_uri_links:
Customizing URI Links in llms.txt
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available.
You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`:
.. code-block:: python
# Default: Link to source files, if _sources exists
llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
# Default: Link to HTML pages instead, if _sources doesn't exist
llms_txt_uri_template = "{base_url}{docname}.html"
# Manual: Link to a custom markdown build
llms_txt_uri_template = "{base_url}{docname}.md"
.. _available_template_variables:
Available Template Variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Your URI template can use the following variables:
- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash)
- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``)
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
.. tip::
Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
See this package's `CMake setup <https://github.com/jdillard/sphinx-llms-txt>`_ for an example of building both HTML and Markdown and/or reStructuredText in parallel.
Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
.. _integration_examples: .. _integration_examples:
Integration Examples Integration Examples
@@ -285,6 +321,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
This is a comprehensive documentation set for our project. This is a comprehensive documentation set for our project.
It includes API references, usage examples, and tutorials. It includes API references, usage examples, and tutorials.
""" """
llms_txt_uri_template = "{base_url}{docname}.md"
# Path handling # Path handling
html_baseurl = "https://docs.example.com/" html_baseurl = "https://docs.example.com/"
+9 -1
View File
@@ -15,12 +15,18 @@ import subprocess
project = "sphinx-llms-txt" project = "sphinx-llms-txt"
copyright = "Jared Dillard" copyright = "Jared Dillard"
author = "Jared Dillard" author = "Jared Dillard"
llms_txt_uri_template = "{base_url}{docname}.md"
llms_txt_full_file = False
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"] llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
llms_txt_summary = """ llms_txt_summary = """
A Sphinx extension that generates a summary llms.txt file,written in Markdown, A Sphinx extension that generates a summary llms.txt file,written in Markdown,
and a single combined documentation llms-full.txt file, written in reStructuredText. and a single combined documentation llms-full.txt file, written in reStructuredText.
""" """
# This doesn't seem to be supported
# rst_file_suffix = ".html.rst"
# check if the current commit is tagged as a release (vX.Y.Z) # check if the current commit is tagged as a release (vX.Y.Z)
try: try:
GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -49,6 +55,8 @@ extensions = [
"sphinx.ext.intersphinx", "sphinx.ext.intersphinx",
"sphinx_contributors", "sphinx_contributors",
"sphinx_llms_txt", "sphinx_llms_txt",
"sphinxcontrib.restbuilder",
"sphinx_inline_tabs",
] ]
# The language for content autogenerated by Sphinx. Refer to documentation # The language for content autogenerated by Sphinx. Refer to documentation
@@ -88,7 +96,7 @@ html_theme_options = {
"source_directory": "docs/source/", "source_directory": "docs/source/",
} }
html_baseurl = "https://sphinx-llms-txt.readthedocs.org/" html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/"
# -- Options for HTMLHelp output --------------------------------------------- # -- Options for HTMLHelp output ---------------------------------------------
+9
View File
@@ -58,6 +58,15 @@ Project Configuration Values
.. versionadded:: 0.2.0 .. versionadded:: 0.2.0
.. confval:: llms_txt_uri_template
- **Type**: string or ``None``
- **Default**: ``None``
- **Description**: Template string for generating URIs in ``llms.txt``.
See :ref:`customizing_uri_links`.
.. versionadded:: 0.7.0
.. confval:: llms_txt_directives .. confval:: llms_txt_directives
- **Type**: list of strings - **Type**: list of strings
+1 -1
View File
@@ -19,7 +19,7 @@ Local development
.. code-block:: console .. code-block:: console
pip install -e ".[dev]" pip install -e . --group dev
#. Install pre-commit Git hook scripts: #. Install pre-commit Git hook scripts:
+5 -3
View File
@@ -4,15 +4,17 @@ Getting Started
Installation Installation
------------ ------------
Directly install via ``pip`` by using: Directly install by using:
.. tab:: via pip
.. code-block:: bash .. code-block:: bash
pip install sphinx-llms-txt pip install sphinx-llms-txt
Or with ``conda`` via ``conda-forge``: .. tab:: via conda:
.. code:: .. code-block:: bash
conda install -c conda-forge sphinx-llms-txt conda install -c conda-forge sphinx-llms-txt
+6 -2
View File
@@ -26,13 +26,16 @@ classifiers = [
license = {text = "MIT"} license = {text = "MIT"}
readme = "README.md" readme = "README.md"
dynamic = ["version"] dynamic = ["version"]
dependencies = [
"sphinx",
]
[project.urls] [project.urls]
download = "https://pypi.org/project/sphinx-llms-txt/" download = "https://pypi.org/project/sphinx-llms-txt/"
source = "https://github.com/jdillard/sphinx-llms-txt" source = "https://github.com/jdillard/sphinx-llms-txt"
changelog = "https://github.com/jdillard/sphinx-llms-txt/blob/master/CHANGELOG.rst" changelog = "https://github.com/jdillard/sphinx-llms-txt/blob/master/CHANGELOG.rst"
[project.optional-dependencies] [dependency-groups]
dev = [ dev = [
"pytest>=7.0.0", "pytest>=7.0.0",
"black", "black",
@@ -40,12 +43,13 @@ dev = [
"mypy", "mypy",
"isort", "isort",
"pre-commit", "pre-commit",
"sphinx",
] ]
test = [ test = [
"pytest>=7.0.0", "pytest>=7.0.0",
] ]
[tool.setuptools]
packages = ["sphinx_llms_txt"]
[tool.setuptools.dynamic] [tool.setuptools.dynamic]
version = {attr = "sphinx_llms_txt.__version__"} version = {attr = "sphinx_llms_txt.__version__"}
+4 -2
View File
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
from .processor import DocumentProcessor from .processor import DocumentProcessor
from .writer import FileWriter from .writer import FileWriter
__version__ = "0.5.1" __version__ = "0.7.0"
# Export classes needed by tests # Export classes needed by tests
__all__ = [ __all__ = [
@@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception):
config = { config = {
"llms_txt_file": app.config.llms_txt_file, "llms_txt_file": app.config.llms_txt_file,
"llms_txt_filename": app.config.llms_txt_filename, "llms_txt_filename": app.config.llms_txt_filename,
"llms_txt_uri_template": app.config.llms_txt_uri_template,
"llms_txt_title": app.config.llms_txt_title, "llms_txt_title": app.config.llms_txt_title,
"llms_txt_summary": summary, "llms_txt_summary": summary,
"llms_txt_full_file": app.config.llms_txt_full_file, "llms_txt_full_file": app.config.llms_txt_full_file,
@@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
app.add_config_value("llms_txt_file", True, "env") app.add_config_value("llms_txt_file", True, "env")
app.add_config_value("llms_txt_filename", "llms.txt", "env") app.add_config_value("llms_txt_filename", "llms.txt", "env")
app.add_config_value("llms_txt_uri_template", None, "env")
app.add_config_value("llms_txt_full_file", True, "env") app.add_config_value("llms_txt_full_file", True, "env")
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env") app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_full_max_size", None, "env") app.add_config_value("llms_txt_full_max_size", None, "env")
@@ -129,7 +131,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
def builder_inited(app): def builder_inited(app):
"""Used to limit what builders are allowed to run the extension.""" """Used to limit what builders are allowed to run the extension."""
allowed_builders = ["html", "singlehtml", "dirhtml"] allowed_builders = ["html", "dirhtml"]
if hasattr(app, "builder") and app.builder.name in allowed_builders: if hasattr(app, "builder") and app.builder.name in allowed_builders:
# Reset manager and root paragraph for each build # Reset manager and root paragraph for each build
global _manager, _root_first_paragraph global _manager, _root_first_paragraph
+35 -13
View File
@@ -197,7 +197,6 @@ class LLMSFullManager:
possible_sources = [ possible_sources = [
Path(outdir) / "_sources", Path(outdir) / "_sources",
Path(outdir) / "html" / "_sources", Path(outdir) / "html" / "_sources",
Path(outdir) / "singlehtml" / "_sources",
] ]
for path in possible_sources: for path in possible_sources:
@@ -205,25 +204,43 @@ class LLMSFullManager:
sources_dir = path sources_dir = path
break break
if not sources_dir: # Get the correct page order (with or without source suffixes)
logger.warning(
"Could not find _sources directory, skipping llms-full creation"
)
return
# Get the correct page order with source suffixes
page_order = self.collector.get_page_order(sources_dir) page_order = self.collector.get_page_order(sources_dir)
if not page_order: if not page_order:
logger.warning( logger.warning("Could not determine page order, skipping file generation")
"Could not determine page order, skipping llms-full creation"
)
return return
# Apply exclusion filter if configured # Apply exclusion filter if configured
page_order = self.collector.filter_excluded_pages(page_order) page_order = self.collector.filter_excluded_pages(page_order)
# Determine output file name and location # If no sources directory, only generate llms.txt and return early
if not sources_dir:
# Generate llms.txt if requested
if self.config.get("llms_txt_file"):
filtered_page_order = self._filter_ignored_pages(page_order)
self.writer.write_verbose_info_to_file(
filtered_page_order,
self.collector.page_titles,
0, # No line count since no llms-full.txt
sources_dir,
)
# Only warn if user explicitly wants llms-full.txt
if self.config.get("llms_txt_full_file"):
# Check if html_copy_source is False
if self.app and not self.app.config.html_copy_source:
logger.warning(
"Could not find _sources directory, skipping llms-full.txt."
"Set html_copy_source = True in conf.py to enable."
)
else:
logger.warning(
"Could not find _sources directory, skipping llms-full.txt"
)
return
# Determine output file name and location for llms-full.txt
output_filename = self.config.get("llms_txt_full_filename") output_filename = self.config.get("llms_txt_full_filename")
output_path = Path(outdir) / output_filename output_path = Path(outdir) / output_filename
@@ -507,6 +524,7 @@ class LLMSFullManager:
filtered_page_order, filtered_page_order,
self.collector.page_titles, self.collector.page_titles,
total_line_count, total_line_count,
sources_dir,
) )
return return
elif action == "note": elif action == "note":
@@ -520,6 +538,7 @@ class LLMSFullManager:
filtered_page_order, filtered_page_order,
self.collector.page_titles, self.collector.page_titles,
total_line_count, total_line_count,
sources_dir,
) )
return return
elif action == "keep": elif action == "keep":
@@ -538,7 +557,10 @@ class LLMSFullManager:
if success and self.config.get("llms_txt_file"): if success and self.config.get("llms_txt_file"):
filtered_page_order = self._filter_ignored_pages(page_order) filtered_page_order = self._filter_ignored_pages(page_order)
self.writer.write_verbose_info_to_file( self.writer.write_verbose_info_to_file(
filtered_page_order, self.collector.page_titles, total_line_count filtered_page_order,
self.collector.page_titles,
total_line_count,
sources_dir,
) )
def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]: def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:
+63 -2
View File
@@ -19,6 +19,42 @@ class FileWriter:
self.outdir = outdir self.outdir = outdir
self.app = app self.app = app
def _resolve_uri_template(self, sources_dir: Path = None) -> str:
"""Resolve which URI template to use based on configuration and sources_dir.
Args:
sources_dir: Path to _sources directory (None if not found)
Returns:
The template string to use for generating URIs
"""
# If custom template exists
custom_template = self.config.get("llms_txt_uri_template")
if custom_template:
# Validate user's template by checking for valid variable names
try:
# Try formatting with test valid values to validate syntax
test_values = {
"base_url": "http://example.com/",
"docname": "test",
"suffix": ".rst",
"sourcelink_suffix": ".txt",
}
custom_template.format(**test_values)
return custom_template
except (KeyError, ValueError) as e:
logger.warning(
f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. "
f"Falling back to default."
)
# Else, use one of the default templates
if sources_dir:
return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
else:
return "{base_url}{docname}.html"
def write_combined_file( def write_combined_file(
self, content_parts: List[str], output_path: Path, total_line_count: int self, content_parts: List[str], output_path: Path, total_line_count: int
) -> bool: ) -> bool:
@@ -50,6 +86,7 @@ class FileWriter:
page_order: Union[List[str], List[Tuple[str, str]]], page_order: Union[List[str], List[Tuple[str, str]]],
page_titles: Dict[str, str], page_titles: Dict[str, str],
total_line_count: int = 0, total_line_count: int = 0,
sources_dir: Path = None,
) -> bool: ) -> bool:
"""Write summary information to the llms.txt file. """Write summary information to the llms.txt file.
@@ -57,6 +94,7 @@ class FileWriter:
page_order: Ordered list of document names or (docname, suffix) tuples page_order: Ordered list of document names or (docname, suffix) tuples
page_titles: Dictionary mapping docnames to titles page_titles: Dictionary mapping docnames to titles
total_line_count: Total number of lines in the combined content total_line_count: Total number of lines in the combined content
sources_dir: Path to _sources directory (None if not found)
Returns: Returns:
True if successful, False otherwise True if successful, False otherwise
@@ -102,14 +140,37 @@ class FileWriter:
if not base_url.endswith("/"): if not base_url.endswith("/"):
base_url += "/" base_url += "/"
# Get sourcelink suffix from Sphinx config
sourcelink_suffix = ""
if self.app and hasattr(self.app.config, "html_sourcelink_suffix"):
sourcelink_suffix = self.app.config.html_sourcelink_suffix
# Handle empty string case specially
if sourcelink_suffix == "":
sourcelink_suffix = "" # Keep it empty
elif not sourcelink_suffix.startswith("."):
sourcelink_suffix = "." + sourcelink_suffix
# Resolve which template to use
uri_template = self._resolve_uri_template(sources_dir)
for item in page_order: for item in page_order:
# Handle both old format (str) and new format (tuple) # Handle both old format (str) and new format (tuple)
if isinstance(item, tuple): if isinstance(item, tuple):
docname, _ = item docname, suffix = item
else: else:
docname = item docname = item
suffix = None
title = page_titles.get(docname, docname) title = page_titles.get(docname, docname)
f.write(f"- [{title}]({base_url}{docname}.html)\n")
uri = uri_template.format(
base_url=base_url,
docname=docname,
suffix=suffix or "",
sourcelink_suffix=sourcelink_suffix,
)
f.write(f"- [{title}]({uri})\n")
logger.info(f"sphinx-llms-txt: created {output_path}") logger.info(f"sphinx-llms-txt: created {output_path}")
return True return True
+126
View File
@@ -794,6 +794,7 @@ def test_summary_default_uses_first_paragraph():
llms_txt_summary = None # Not configured llms_txt_summary = None # Not configured
llms_txt_file = True llms_txt_file = True
llms_txt_filename = "llms.txt" llms_txt_filename = "llms.txt"
llms_txt_uri_template = None
llms_txt_title = None llms_txt_title = None
llms_txt_full_file = True llms_txt_full_file = True
llms_txt_full_filename = "llms-full.txt" llms_txt_full_filename = "llms-full.txt"
@@ -1031,3 +1032,128 @@ def test_code_files_ignored_patterns(tmp_path, caplog):
assert ( assert (
"Code file pattern 'docs/**/*.rst' ignored." in captured_warnings[0] "Code file pattern 'docs/**/*.rst' ignored." in captured_warnings[0]
), f"Warning message should contain expected text. Got: {captured_warnings[0]}" ), f"Warning message should contain expected text. Got: {captured_warnings[0]}"
def test_llms_txt_generated_without_sources_dir(tmp_path):
"""Test that llms.txt is generated even when _sources directory doesn't exist."""
from sphinx_llms_txt.manager import LLMSFullManager
# Create manager
manager = LLMSFullManager()
# Set config to enable llms.txt
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
"llms_txt_full_file": True,
"llms_txt_full_filename": "llms-full.txt",
"llms_txt_exclude": [],
"llms_txt_directives": [],
}
manager.set_config(config)
# Create directories (but no _sources)
outdir = tmp_path / "build"
srcdir = tmp_path / "source"
outdir.mkdir()
srcdir.mkdir()
# Mock env with documents
class MockEnv:
all_docs = {"index": None, "about": None}
titles = {
"index": type("TitleNode", (), {"astext": lambda self: "Home"})(),
"about": type("TitleNode", (), {"astext": lambda self: "About"})(),
}
toctree_includes = {"index": ["about"]}
manager.set_env(MockEnv())
manager.set_master_doc("index")
# Update page titles directly in the collector
manager.update_page_title("index", "Home")
manager.update_page_title("about", "About")
# Call combine_sources - should generate llms.txt even without _sources
manager.combine_sources(str(outdir), str(srcdir))
# Verify llms.txt was created
llms_txt = outdir / "llms.txt"
assert llms_txt.exists(), "llms.txt should be generated even without _sources"
# Verify llms-full.txt was NOT created (since no _sources)
llms_full_txt = outdir / "llms-full.txt"
assert (
not llms_full_txt.exists()
), "llms-full.txt should not be generated without _sources"
# Read llms.txt and verify it has content
with open(llms_txt, "r", encoding="utf-8") as f:
content = f.read()
# Should contain page titles and links
assert "Home" in content
assert "About" in content
assert "index.html" in content
assert "about.html" in content
def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
"""
Test that no warning is logged when llms_txt_full_file=False and
_sources doesn't exist.
"""
from unittest.mock import patch
from sphinx_llms_txt.manager import LLMSFullManager
# Create manager
manager = LLMSFullManager()
# Set config with llms_txt_full_file=False
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
"llms_txt_full_file": False, # User doesn't want llms-full.txt
"llms_txt_full_filename": "llms-full.txt",
"llms_txt_exclude": [],
"llms_txt_directives": [],
}
manager.set_config(config)
# Create directories (but no _sources)
outdir = tmp_path / "build"
srcdir = tmp_path / "source"
outdir.mkdir()
srcdir.mkdir()
# Mock env with documents
class MockEnv:
all_docs = {"index": None}
titles = {"index": type("TitleNode", (), {"astext": lambda self: "Home"})()}
toctree_includes = {"index": []}
manager.set_env(MockEnv())
manager.set_master_doc("index")
manager.update_page_title("index", "Home")
# Capture warnings
captured_warnings = []
def capture_warning(message, *args, **kwargs):
if "_sources" in str(message):
captured_warnings.append(message)
with patch("sphinx_llms_txt.manager.logger.warning", side_effect=capture_warning):
# Call combine_sources
manager.combine_sources(str(outdir), str(srcdir))
# Verify NO warning was logged since llms_txt_full_file=False
assert (
len(captured_warnings) == 0
), "No warning should be logged when llms_txt_full_file=False"
# Verify llms.txt was still created
llms_txt = outdir / "llms.txt"
assert llms_txt.exists()
+175
View File
@@ -0,0 +1,175 @@
"""Test URI template functionality for llms.txt links."""
from sphinx_llms_txt import FileWriter
def test_uri_template_with_sources_dir(tmp_path):
"""Test that default template uses _sources links when sources_dir exists."""
build_dir = tmp_path / "build"
build_dir.mkdir()
# Create _sources directory to simulate its existence
sources_dir = build_dir / "_sources"
sources_dir.mkdir()
# Mock app with html_sourcelink_suffix
class MockApp:
class Config:
html_sourcelink_suffix = ".txt"
config = Config()
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
"llms_txt_uri_template": (
"{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
),
"html_baseurl": "https://example.com",
}
writer = FileWriter(config, str(build_dir), MockApp())
page_titles = {
"index": "Home Page",
"about": "About Us",
}
# Page order with suffixes (simulating _sources files exist)
page_order = [("index", ".rst"), ("about", ".md")]
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
# Check that the file was created
verbose_file = build_dir / "llms.txt"
assert verbose_file.exists()
# Read the file content
with open(verbose_file, "r", encoding="utf-8") as f:
content = f.read()
# Should link to _sources files
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
assert "- [About Us](https://example.com/_sources/about.md.txt)" in content
def test_uri_template_without_sources_dir(tmp_path):
"""
Test that HTML template is used when sources_dir doesn't exist and no custom
template.
"""
build_dir = tmp_path / "build"
build_dir.mkdir()
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
# No custom template set
"html_baseurl": "https://example.com",
}
writer = FileWriter(config, str(build_dir))
page_titles = {
"index": "Home Page",
"about": "About Us",
}
# Page order without suffixes (simulating no _sources)
page_order = [("index", None), ("about", None)]
# Pass None for sources_dir to simulate it doesn't exist
writer.write_verbose_info_to_file(page_order, page_titles, 0, None)
# Check that the file was created
verbose_file = build_dir / "llms.txt"
assert verbose_file.exists()
# Read the file content
with open(verbose_file, "r", encoding="utf-8") as f:
content = f.read()
# Should fallback to HTML links
assert "- [Home Page](https://example.com/index.html)" in content
assert "- [About Us](https://example.com/about.html)" in content
def test_uri_template_custom(tmp_path):
"""Test that custom URI template works correctly."""
build_dir = tmp_path / "build"
build_dir.mkdir()
sources_dir = build_dir / "_sources"
sources_dir.mkdir()
# Mock app with html_sourcelink_suffix
class MockApp:
class Config:
html_sourcelink_suffix = ".txt"
config = Config()
# Custom template that uses different path
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
"llms_txt_uri_template": "{base_url}raw/{docname}{suffix}",
"html_baseurl": "https://example.com/",
}
writer = FileWriter(config, str(build_dir), MockApp())
page_titles = {
"index": "Home Page",
}
page_order = [("index", ".rst")]
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
verbose_file = build_dir / "llms.txt"
with open(verbose_file, "r", encoding="utf-8") as f:
content = f.read()
# Should use custom template
assert "- [Home Page](https://example.com/raw/index.rst)" in content
def test_uri_template_invalid_fallback(tmp_path):
"""
Test that invalid template falls back to default sources template when
sources_dir exists.
"""
build_dir = tmp_path / "build"
build_dir.mkdir()
sources_dir = build_dir / "_sources"
sources_dir.mkdir()
# Mock app with html_sourcelink_suffix
class MockApp:
class Config:
html_sourcelink_suffix = ".txt"
config = Config()
# Invalid template with typo in variable name
config = {
"llms_txt_file": True,
"llms_txt_filename": "llms.txt",
"llms_txt_uri_template": "{base_urll}/{docname}",
"html_baseurl": "https://example.com",
}
writer = FileWriter(config, str(build_dir), MockApp())
page_titles = {
"index": "Home Page",
}
page_order = [("index", ".rst")]
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
verbose_file = build_dir / "llms.txt"
with open(verbose_file, "r", encoding="utf-8") as f:
content = f.read()
# Should fallback to default sources template
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content