Compare commits

..
Author SHA1 Message Date
dependabot[bot]andGitHub 327ae7c2d7 Bump actions/checkout from 6 to 7 in the all-github-actions group
Bumps the all-github-actions group with 1 update: [actions/checkout](https://github.com/actions/checkout).


Updates `actions/checkout` from 6 to 7
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v6...v7)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-07-01 22:26:07 +00:00
Jared Dillard 10794dce65 Add note to docs about sphinx-llm 2026-03-18 23:56:37 -07:00
Jacob TomlinsonandGitHub f6596dd1ec Update sphinx-llm link to NVIDIA repository (#64) 2026-02-05 20:50:40 -08:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
29c3932488 Bump actions/checkout from 5 to 6 in the all-github-actions group (#62)
Bumps the all-github-actions group with 1 update: [actions/checkout](https://github.com/actions/checkout).


Updates `actions/checkout` from 5 to 6
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-01-08 11:34:16 -08:00
Kayce BasquesandGitHub 1dd8119cfd Don't process includes within code blocks (#58) 2025-12-16 14:30:38 -08:00
Jared Dillard dacabd62b4 update modules to 0.3.0 2025-12-11 12:49:49 -08:00
Jared Dillard 2f8e64cc6d Make more generic 2025-12-11 11:58:35 -08:00
Jared Dillard 71dc28e7d8 update to 0.2.0 2025-12-11 01:57:02 -08:00
Jared DillardandGitHub cbbf4b572b [docs] Use FetchContent to grab CMake modules (#61) 2025-12-11 00:55:00 -08:00
Jared DillardandGitHub 8a1a73512d It's parallel!
Updated footnote reference for sphinx-llm to clarify usage.
2025-12-10 21:58:49 -08:00
Jared Dillard 097f2c1084 add note about sphinx-llm 2025-12-08 14:13:24 -08:00
Jared DillardandGitHub 61f9b38d4d [docs] Demo all format options (#60) 2025-12-08 00:40:59 -08:00
Jared DillardandGitHub 8254002622 [docs] Improve docs wording (#59) 2025-12-07 23:26:27 -08:00
Jared Dillard 5b1b72bafb Change markdown suffix 2025-12-07 22:45:31 -08:00
Jared Dillard 3a4ccca5bb Update highlights 2025-12-06 19:38:54 -08:00
Jared DillardandGitHub 0a3da6b1f9 [docs] Add docs on the CMake workflow (#56) 2025-12-05 18:37:24 -08:00
Jared DillardandGitHub f1a3963a0d [docs] Discuss output formats in the docs (#55) 2025-12-03 13:51:09 -08:00
Jared Dillard c23c6d4468 Add tabs 2025-12-02 22:09:14 -08:00
Jared Dillard db32dc60a7 Improve CMake bits 2025-12-02 16:48:29 -08:00
Jared DillardandGitHub 543efabebb [docs] Add restbuilder builder for single page .rst builds (#53) 2025-12-01 13:34:45 -08:00
Jared Dillard 141e0e29f6 clean up the cmake 2025-10-28 12:26:45 -07:00
Jared DillardandGitHub 1b08b2f362 Add docs on markdown usage (#50) 2025-10-27 20:33:28 -07:00
16 changed files with 435 additions and 78 deletions
+2 -2
View File
@@ -10,7 +10,7 @@ jobs:
pre-commit: pre-commit:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v5 - uses: actions/checkout@v7
- name: Set up Python 3.10 - name: Set up Python 3.10
uses: actions/setup-python@v6 uses: actions/setup-python@v6
with: with:
@@ -23,7 +23,7 @@ jobs:
python-version: ['3.9', '3.10', '3.11', '3.12'] python-version: ['3.9', '3.10', '3.11', '3.12']
steps: steps:
- uses: actions/checkout@v5 - uses: actions/checkout@v7
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6 uses: actions/setup-python@v6
+5 -1
View File
@@ -9,6 +9,10 @@ build:
- pip install -r docs/requirements.txt - pip install -r docs/requirements.txt
- pip install -e . - pip install -e .
- cmake --workflow --preset documentation-workflow - cmake --workflow --preset documentation-workflow
# Generate llms.txt variants for demo purposes
- python docs/generate_llms_variants.py build/html
# Copy built documentation to Read the Docs output directory # Copy built documentation to Read the Docs output directory
- mkdir -p $READTHEDOCS_OUTPUT/html - mkdir -p $READTHEDOCS_OUTPUT/html
- cp -r build/* $READTHEDOCS_OUTPUT/html/ - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
- cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
- cp -r build/rst/* $READTHEDOCS_OUTPUT/html/
+5
View File
@@ -1,6 +1,11 @@
Changelog Changelog
========= =========
0.7.1
-----
- Don't process includes within code blocks
0.7.0 0.7.0
----- -----
+10 -3
View File
@@ -1,8 +1,15 @@
cmake_minimum_required(VERSION 3.15) cmake_minimum_required(VERSION 3.15)
project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE) project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
# Add CMake module path # Fetch Sphinx CMake modules
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") include(FetchContent)
FetchContent_Declare(
sphinx_cmake_modules
GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
GIT_TAG main
)
FetchContent_MakeAvailable(sphinx_cmake_modules)
list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
# Add documentation # Add documentation
add_subdirectory(docs) add_subdirectory(docs)
+9 -3
View File
@@ -22,10 +22,16 @@
"targets": ["markdown"] "targets": ["markdown"]
}, },
{ {
"name": "docs-parallel", "name": "rst",
"displayName": "Build HTML and Markdown in parallel", "displayName": "Build reStructuredText Documentation",
"configurePreset": "documentation", "configurePreset": "documentation",
"targets": ["html", "markdown"] "targets": ["rst"]
},
{
"name": "docs-parallel",
"displayName": "Build all output formats in parallel",
"configurePreset": "documentation",
"targets": ["html", "markdown", "rst"]
} }
], ],
"workflowPresets": [ "workflowPresets": [
-27
View File
@@ -1,27 +0,0 @@
# SetupSphinx.cmake - Find Sphinx for CI environments
# Function to find Sphinx in the system
function(setup_sphinx_environment)
# Find sphinx-build executable in system
find_program(SPHINX_EXECUTABLE
NAMES sphinx-build
DOC "Sphinx documentation generator"
)
if(NOT SPHINX_EXECUTABLE)
message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
endif()
# Get Sphinx version
execute_process(
COMMAND ${SPHINX_EXECUTABLE} --version
OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
ERROR_VARIABLE SPHINX_VERSION_OUTPUT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
message(STATUS "${SPHINX_VERSION_OUTPUT}")
# Export to parent scope
set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
endfunction()
+5 -22
View File
@@ -1,24 +1,7 @@
# Find Sphinx include(SphinxUtils)
include(SetupSphinx)
setup_sphinx_environment() setup_sphinx_environment()
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source) add_sphinx_builder(html)
set(SPHINX_BUILD ${CMAKE_BINARY_DIR}) add_sphinx_builder(markdown)
add_sphinx_builder(rst)
# Build HTML documentation
add_custom_target(html
COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
COMMENT "Building HTML documentation"
VERBATIM
)
# Build markdown documentation
add_custom_target(markdown
COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
COMMENT "Building Markdown documentation"
VERBATIM
)
# Build both formats
add_custom_target(docs ALL)
add_dependencies(docs html markdown)
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""
Generate variant llms.txt files for demo purposes.
Takes the generated llms.txt (with _sources links) and creates:
- llms.txt - default with _sources links (unchanged)
- llms.md.txt - .html.md links
- llms.rst.txt - .rst links
"""
import re
import sys
from pathlib import Path
def get_base_url() -> str:
"""Import base_url from conf.py."""
sys.path.insert(0, str(Path(__file__).parent / "source"))
from conf import html_baseurl # noqa: E402
return html_baseurl
def generate_variants(build_dir: Path) -> None:
"""Generate llms.txt variants from the original file."""
original = build_dir / "llms.txt"
if not original.exists():
print(f"Error: {original} not found")
sys.exit(1)
content = original.read_text()
base_url = get_base_url()
# Pattern to match links like: https://.../_sources/{docname}.rst.txt
link_pattern = re.compile(
rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
)
# Generate .html.md variant
md_content = link_pattern.sub(r"\1\2.html.md", content)
(build_dir / "llms.md.txt").write_text(md_content)
print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
# Generate .rst variant
rst_content = link_pattern.sub(r"\1\2.rst", content)
(build_dir / "llms.rst.txt").write_text(rst_content)
print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
if __name__ == "__main__":
if len(sys.argv) > 1:
build_dir = Path(sys.argv[1])
else:
build_dir = Path("build/html")
generate_variants(build_dir)
+3
View File
@@ -2,6 +2,9 @@ furo
esbonio esbonio
sphinx-contributors sphinx-contributors
sphinx sphinx
sphinx-design
sphinx-llms-txt sphinx-llms-txt
sphinx-inline-tabs
sphinxext-opengraph sphinxext-opengraph
sphinx-markdown-builder sphinx-markdown-builder
sphinxcontrib-restbuilder
+141
View File
@@ -292,6 +292,147 @@ Your URI template can use the following variables:
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists - ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``) - ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
.. tip::
Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
.. _cmake_workflow:
CMake Workflow
^^^^^^^^^^^^^^
This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
Key Files
~~~~~~~~~
These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
.. code-block:: text
.
├── .readthedocs.yml
├── CMakeLists.txt
├── CMakePresets.json
└── docs/
└── CMakeLists.txt
Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
.. dropdown:: .readthedocs.yml
:chevron: down-up
A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
.. literalinclude:: ../../.readthedocs.yml
:language: yaml
:lines: 1-9,11,14-
:linenos:
:emphasize-lines: 9, 14-15
.. list-table::
:header-rows: 1
:width: 100%
:widths: 15 85
* - Line
- Description
* - **9**
- Update the path if your requirements file is in a different location
* - **13-14**
- Modify the copy commands for the output formats you deploy
.. dropdown:: CMakeLists.txt
:chevron: down-up
A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
.. literalinclude:: ../../CMakeLists.txt
:language: cmake
:linenos:
:emphasize-lines: 9, 15
.. list-table::
:header-rows: 1
:width: 100%
:widths: 15 85
* - Line
- Description
* - **9**
- Update the ``GIT_TAG`` to use a different version or commit hash
* - **15**
- Change if your docs subdirectory has a different location
.. dropdown:: docs/CMakeLists.txt
:chevron: down-up
A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
.. literalinclude:: ../CMakeLists.txt
:language: cmake
:linenos:
:emphasize-lines: 5-7
.. list-table::
:header-rows: 1
:width: 100%
:widths: 15 85
* - Line
- Description
* - **5-7**
- Add or remove calls based on which output formats you need
.. dropdown:: CMakePresets.json
:chevron: down-up
Defines presets for configuring and building documentation:
- **Configure Presets:** Sets up the build directory.
- **Build Presets:** Defines build formats individually and all in parallel.
- **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
.. literalinclude:: ../../CMakePresets.json
:language: json
:linenos:
:emphasize-lines: 18-23, 24-29, 34
.. list-table::
:header-rows: 1
:width: 100%
:widths: 15 85
* - Line
- Description
* - **18-23**
- Remove this preset to disable Markdown documentation builds
* - **24-29**
- Remove this preset to disable reStructuredText documentation builds
* - **34**
- Modify the targets list to build only the output formats you need in parallel
Usage
~~~~~
To build documentation locally using CMake:
.. code-block:: console
# Run the full workflow (configure + build all formats)
cmake --workflow --preset documentation-workflow
# Or configure and build separately
cmake --preset documentation
cmake --build --preset html # Build HTML only
cmake --build --preset docs-parallel # Build all formats
.. _integration_examples: .. _integration_examples:
Integration Examples Integration Examples
+9 -1
View File
@@ -15,13 +15,17 @@ import subprocess
project = "sphinx-llms-txt" project = "sphinx-llms-txt"
copyright = "Jared Dillard" copyright = "Jared Dillard"
author = "Jared Dillard" author = "Jared Dillard"
llms_txt_uri_template = "{base_url}{docname}.md"
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"] llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
llms_txt_summary = """ llms_txt_summary = """
A Sphinx extension that generates a summary llms.txt file,written in Markdown, A Sphinx extension that generates a summary llms.txt file,written in Markdown,
and a single combined documentation llms-full.txt file, written in reStructuredText. and a single combined documentation llms-full.txt file, written in reStructuredText.
""" """
# This doesn't seem to be supported
# rst_file_suffix = ".html.rst"
markdown_file_suffix = ".html.md"
# check if the current commit is tagged as a release (vX.Y.Z) # check if the current commit is tagged as a release (vX.Y.Z)
try: try:
GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"]) GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -50,6 +54,10 @@ extensions = [
"sphinx.ext.intersphinx", "sphinx.ext.intersphinx",
"sphinx_contributors", "sphinx_contributors",
"sphinx_llms_txt", "sphinx_llms_txt",
"sphinxcontrib.restbuilder",
"sphinx_inline_tabs",
"sphinx.ext.extlinks",
"sphinx_design",
] ]
# The language for content autogenerated by Sphinx. Refer to documentation # The language for content autogenerated by Sphinx. Refer to documentation
+59 -5
View File
@@ -4,15 +4,17 @@ Getting Started
Installation Installation
------------ ------------
Directly install via ``pip`` by using: Directly install by using:
.. tab:: via pip
.. code-block:: bash .. code-block:: bash
pip install sphinx-llms-txt pip install sphinx-llms-txt
Or with ``conda`` via ``conda-forge``: .. tab:: via conda:
.. code:: .. code-block:: bash
conda install -c conda-forge sphinx-llms-txt conda install -c conda-forge sphinx-llms-txt
@@ -32,7 +34,59 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
sphinx-llms-txt: created /path/to/_build/html/llms.txt sphinx-llms-txt: created /path/to/_build/html/llms.txt
.. _choosing-output-format:
.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades. Choosing an Output Format
-------------------------
By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
.. list-table:: Output Format Comparison
:header-rows: 1
:widths: 18 27 27 27
* -
- Default
- Markdown
- reStructuredText
* - **Setup**
- No config
- CMake [#sphinxllm]_
- CMake
* - **Builder**
- Native [#native]_
- `sphinx-markdown-builder`_
- `sphinxcontrib-restbuilder`_
* - **Format**
- Raw reStructuredText source
- Rendered Markdown [#rendered]_
- Rendered reStructuredText [#rendered]_
* - **LLM Readability**
- Good - preserves structure for simple syntax
- Excellent - native LLM format
- Good - Can provide more structured content
* - **Key Advantage**
- Zero setup required
- More compact (less input tokens)
- Can preserve Sphinx semantics
* - **Key Disadvantage**
- Raw directives won't be parsed [#autodoc]_
- Loses structure from complex directives
- Can lose structure from complex directives
* - **llms-full.txt support**
- Supported with above caveats
- Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
- Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
.. rubric:: Footnotes
.. [#sphinxllm] See `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.
+18 -9
View File
@@ -8,21 +8,28 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
Demo Demo
---- ----
You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example. This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
Highlights Highlights
---------- ----------
1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build **Zero Configuration**
2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content Add the extension to your ``conf.py`` and you're done.
3. **Path Resolution**: Transforms relative paths in directives to full paths The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
4. **Output Generation**: Creates two optional files:
- ``llms.txt``: A concise summary of your documentation, in Markdown **Intelligent Content Processing**
- ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
5. **Content Filtering**: Allows you to exclude specific pages or sections **Customizable When Needed**
6. **Source Code**: Allows you to include specific source code files Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
.. seealso::
For better default output without configuration, see `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ from NVIDIA.
sphinx-llms-txt is best when customized with alternative output formats, content filtering, or source code inclusion.
.. toctree:: .. toctree::
:maxdepth: 2 :maxdepth: 2
@@ -36,6 +43,8 @@ Highlights
.. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt .. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
.. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt .. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
.. _Sphinx: http://sphinx-doc.org/ .. _Sphinx: http://sphinx-doc.org/
.. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg .. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg
+1 -1
View File
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
from .processor import DocumentProcessor from .processor import DocumentProcessor
from .writer import FileWriter from .writer import FileWriter
__version__ = "0.7.0" __version__ = "0.7.1"
# Export classes needed by tests # Export classes needed by tests
__all__ = [ __all__ = [
+81 -1
View File
@@ -128,8 +128,11 @@ class DocumentProcessor:
Returns: Returns:
Processed content with directive paths properly resolved Processed content with directive paths properly resolved
""" """
# Get code block ranges to skip directives inside them
code_block_ranges = self._get_code_block_ranges(content)
# Get the configured path directives to process # Get the configured path directives to process
default_path_directives = ["image", "figure"] default_path_directives = ["image", "figure", "literalinclude"]
custom_path_directives = self.config.get("llms_txt_directives") custom_path_directives = self.config.get("llms_txt_directives")
path_directives = set(default_path_directives + custom_path_directives) path_directives = set(default_path_directives + custom_path_directives)
@@ -143,6 +146,11 @@ class DocumentProcessor:
is_test = "pytest" in str(source_path) and "subdir" in str(source_path) is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
def replace_directive_path(match, base_url=base_url, is_test=is_test): def replace_directive_path(match, base_url=base_url, is_test=is_test):
# Check if this directive is within a code block
if self._is_in_code_block(match.start(), code_block_ranges):
# This directive is inside a code block, don't process it
return match.group(0)
prefix = match.group(1) # The entire directive prefix including whitespace prefix = match.group(1) # The entire directive prefix including whitespace
path = match.group(3).strip() # The path argument path = match.group(3).strip() # The path argument
@@ -276,6 +284,71 @@ class DocumentProcessor:
return possible_paths return possible_paths
def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
"""Find all code block ranges in the content.
Args:
content: The source content to analyze
Returns:
List of (start, end) tuples representing code block character
ranges
"""
code_block_ranges = []
# Match code block as well as `code` and `sourcecode` aliases
code_block_pattern = re.compile(
r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
)
for match in code_block_pattern.finditer(content):
start_pos = match.start()
indent = match.group(1)
indent_len = len(indent)
# Find the end of the code block by looking for the next line
# that is not indented more than the directive
block_start = match.end()
pos = block_start
# Skip any blank lines immediately after the directive
while pos < len(content) and content[pos] in "\n":
pos += 1
# Find where the code block ends
lines = content[pos:].split("\n")
block_end = pos
for line in lines:
if line.strip(): # Non-empty line
# Check indentation level
line_indent = len(line) - len(line.lstrip())
if line_indent <= indent_len:
# The block ends when we find a line that is indented
# less than the directive itself
break
block_end += len(line) + 1 # +1 for the newline
code_block_ranges.append((start_pos, block_end))
return code_block_ranges
def _is_in_code_block(
self, match_start: int, code_block_ranges: List[Tuple[int, int]]
) -> bool:
"""Check if a match position is within a code block.
Args:
match_start: The starting position of the match
code_block_ranges: List of (start, end) tuples for code blocks
Returns:
True if the match is within a code block, False otherwise
"""
for block_start, block_end in code_block_ranges:
if block_start <= match_start < block_end:
return True
return False
def _process_includes(self, content: str, source_path: Path) -> str: def _process_includes(self, content: str, source_path: Path) -> str:
"""Process include directives in content. """Process include directives in content.
@@ -286,11 +359,18 @@ class DocumentProcessor:
Returns: Returns:
Processed content with include directives replaced with included content Processed content with include directives replaced with included content
""" """
code_block_ranges = self._get_code_block_ranges(content)
# Find all include directives using regex # Find all include directives using regex
include_pattern = build_directive_pattern(["include"]) include_pattern = build_directive_pattern(["include"])
# Function to replace each include with content # Function to replace each include with content
def replace_include(match): def replace_include(match):
# Check if this include is within a code block
if self._is_in_code_block(match.start(), code_block_ranges):
# This include is inside a code block, don't process it
return match.group(0)
include_path = match.group(3) include_path = match.group(3)
directive_part = match.group( directive_part = match.group(
1 1
+25
View File
@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
assert processed_content == expected_content assert processed_content == expected_content
def test_process_includes_in_code_block(tmp_path):
"""Test that an `include` within a `code-block` is not processed."""
# Create a processor
config = {"llms_txt_directives": []}
processor = DocumentProcessor(config)
# Create a source file that uses include syntax within a `code-block`
source_content = (
"Normal paragraph.\n\n"
".. code-block:: rst\n\n"
" .. include:: foo.txt\n\n"
"Another normal paragraph."
)
source_file = tmp_path / "source.txt"
with open(source_file, "w", encoding="utf-8") as f:
f.write(source_content)
# Run the include directive processor
processed_content = processor._process_includes(source_content, source_file)
# Check that the include directive was not processed
expected_content = source_content
assert processed_content == expected_content
def test_process_includes_with_relative_paths(tmp_path): def test_process_includes_with_relative_paths(tmp_path):
"""Test that include directives with relative paths are processed correctly.""" """Test that include directives with relative paths are processed correctly."""
# Create a processor # Create a processor