Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
327ae7c2d7 | ||
|
|
10794dce65 | ||
|
|
f6596dd1ec | ||
|
|
29c3932488 | ||
|
|
1dd8119cfd | ||
|
|
dacabd62b4 | ||
|
|
2f8e64cc6d | ||
|
|
71dc28e7d8 | ||
|
|
cbbf4b572b | ||
|
|
8a1a73512d | ||
|
|
097f2c1084 | ||
|
|
61f9b38d4d | ||
|
|
8254002622 | ||
|
|
5b1b72bafb | ||
|
|
3a4ccca5bb | ||
|
|
0a3da6b1f9 | ||
|
|
f1a3963a0d | ||
|
|
c23c6d4468 | ||
|
|
db32dc60a7 | ||
|
|
543efabebb | ||
|
|
141e0e29f6 | ||
|
|
1b08b2f362 |
@@ -10,7 +10,7 @@ jobs:
|
|||||||
pre-commit:
|
pre-commit:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v7
|
||||||
- name: Set up Python 3.10
|
- name: Set up Python 3.10
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
@@ -23,7 +23,7 @@ jobs:
|
|||||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v5
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v6
|
||||||
|
|||||||
+5
-1
@@ -9,6 +9,10 @@ build:
|
|||||||
- pip install -r docs/requirements.txt
|
- pip install -r docs/requirements.txt
|
||||||
- pip install -e .
|
- pip install -e .
|
||||||
- cmake --workflow --preset documentation-workflow
|
- cmake --workflow --preset documentation-workflow
|
||||||
|
# Generate llms.txt variants for demo purposes
|
||||||
|
- python docs/generate_llms_variants.py build/html
|
||||||
# Copy built documentation to Read the Docs output directory
|
# Copy built documentation to Read the Docs output directory
|
||||||
- mkdir -p $READTHEDOCS_OUTPUT/html
|
- mkdir -p $READTHEDOCS_OUTPUT/html
|
||||||
- cp -r build/* $READTHEDOCS_OUTPUT/html/
|
- cp -r build/html/* $READTHEDOCS_OUTPUT/html/
|
||||||
|
- cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
|
||||||
|
- cp -r build/rst/* $READTHEDOCS_OUTPUT/html/
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
Changelog
|
Changelog
|
||||||
=========
|
=========
|
||||||
|
|
||||||
|
0.7.1
|
||||||
|
-----
|
||||||
|
|
||||||
|
- Don't process includes within code blocks
|
||||||
|
|
||||||
0.7.0
|
0.7.0
|
||||||
-----
|
-----
|
||||||
|
|
||||||
|
|||||||
+10
-3
@@ -1,8 +1,15 @@
|
|||||||
cmake_minimum_required(VERSION 3.15)
|
cmake_minimum_required(VERSION 3.15)
|
||||||
project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
|
project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
|
||||||
|
|
||||||
# Add CMake module path
|
# Fetch Sphinx CMake modules
|
||||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
include(FetchContent)
|
||||||
|
FetchContent_Declare(
|
||||||
|
sphinx_cmake_modules
|
||||||
|
GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
|
||||||
|
GIT_TAG main
|
||||||
|
)
|
||||||
|
FetchContent_MakeAvailable(sphinx_cmake_modules)
|
||||||
|
list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
|
||||||
|
|
||||||
# Add documentation
|
# Add documentation
|
||||||
add_subdirectory(docs)
|
add_subdirectory(docs)
|
||||||
|
|||||||
+9
-3
@@ -22,10 +22,16 @@
|
|||||||
"targets": ["markdown"]
|
"targets": ["markdown"]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "docs-parallel",
|
"name": "rst",
|
||||||
"displayName": "Build HTML and Markdown in parallel",
|
"displayName": "Build reStructuredText Documentation",
|
||||||
"configurePreset": "documentation",
|
"configurePreset": "documentation",
|
||||||
"targets": ["html", "markdown"]
|
"targets": ["rst"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "docs-parallel",
|
||||||
|
"displayName": "Build all output formats in parallel",
|
||||||
|
"configurePreset": "documentation",
|
||||||
|
"targets": ["html", "markdown", "rst"]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"workflowPresets": [
|
"workflowPresets": [
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
# SetupSphinx.cmake - Find Sphinx for CI environments
|
|
||||||
|
|
||||||
# Function to find Sphinx in the system
|
|
||||||
function(setup_sphinx_environment)
|
|
||||||
# Find sphinx-build executable in system
|
|
||||||
find_program(SPHINX_EXECUTABLE
|
|
||||||
NAMES sphinx-build
|
|
||||||
DOC "Sphinx documentation generator"
|
|
||||||
)
|
|
||||||
|
|
||||||
if(NOT SPHINX_EXECUTABLE)
|
|
||||||
message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
# Get Sphinx version
|
|
||||||
execute_process(
|
|
||||||
COMMAND ${SPHINX_EXECUTABLE} --version
|
|
||||||
OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
|
|
||||||
ERROR_VARIABLE SPHINX_VERSION_OUTPUT
|
|
||||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
|
||||||
)
|
|
||||||
message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
|
|
||||||
message(STATUS "${SPHINX_VERSION_OUTPUT}")
|
|
||||||
|
|
||||||
# Export to parent scope
|
|
||||||
set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
|
|
||||||
endfunction()
|
|
||||||
+5
-22
@@ -1,24 +1,7 @@
|
|||||||
# Find Sphinx
|
include(SphinxUtils)
|
||||||
include(SetupSphinx)
|
|
||||||
setup_sphinx_environment()
|
setup_sphinx_environment()
|
||||||
|
|
||||||
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
|
add_sphinx_builder(html)
|
||||||
set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
|
add_sphinx_builder(markdown)
|
||||||
|
add_sphinx_builder(rst)
|
||||||
# Build HTML documentation
|
|
||||||
add_custom_target(html
|
|
||||||
COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
|
||||||
COMMENT "Building HTML documentation"
|
|
||||||
VERBATIM
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build markdown documentation
|
|
||||||
add_custom_target(markdown
|
|
||||||
COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
|
||||||
COMMENT "Building Markdown documentation"
|
|
||||||
VERBATIM
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build both formats
|
|
||||||
add_custom_target(docs ALL)
|
|
||||||
add_dependencies(docs html markdown)
|
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Generate variant llms.txt files for demo purposes.
|
||||||
|
|
||||||
|
Takes the generated llms.txt (with _sources links) and creates:
|
||||||
|
- llms.txt - default with _sources links (unchanged)
|
||||||
|
- llms.md.txt - .html.md links
|
||||||
|
- llms.rst.txt - .rst links
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def get_base_url() -> str:
|
||||||
|
"""Import base_url from conf.py."""
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent / "source"))
|
||||||
|
from conf import html_baseurl # noqa: E402
|
||||||
|
|
||||||
|
return html_baseurl
|
||||||
|
|
||||||
|
|
||||||
|
def generate_variants(build_dir: Path) -> None:
|
||||||
|
"""Generate llms.txt variants from the original file."""
|
||||||
|
original = build_dir / "llms.txt"
|
||||||
|
|
||||||
|
if not original.exists():
|
||||||
|
print(f"Error: {original} not found")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
content = original.read_text()
|
||||||
|
base_url = get_base_url()
|
||||||
|
|
||||||
|
# Pattern to match links like: https://.../_sources/{docname}.rst.txt
|
||||||
|
link_pattern = re.compile(
|
||||||
|
rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Generate .html.md variant
|
||||||
|
md_content = link_pattern.sub(r"\1\2.html.md", content)
|
||||||
|
(build_dir / "llms.md.txt").write_text(md_content)
|
||||||
|
print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
|
||||||
|
|
||||||
|
# Generate .rst variant
|
||||||
|
rst_content = link_pattern.sub(r"\1\2.rst", content)
|
||||||
|
(build_dir / "llms.rst.txt").write_text(rst_content)
|
||||||
|
print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
|
||||||
|
|
||||||
|
print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
build_dir = Path(sys.argv[1])
|
||||||
|
else:
|
||||||
|
build_dir = Path("build/html")
|
||||||
|
|
||||||
|
generate_variants(build_dir)
|
||||||
@@ -2,6 +2,9 @@ furo
|
|||||||
esbonio
|
esbonio
|
||||||
sphinx-contributors
|
sphinx-contributors
|
||||||
sphinx
|
sphinx
|
||||||
|
sphinx-design
|
||||||
sphinx-llms-txt
|
sphinx-llms-txt
|
||||||
|
sphinx-inline-tabs
|
||||||
sphinxext-opengraph
|
sphinxext-opengraph
|
||||||
sphinx-markdown-builder
|
sphinx-markdown-builder
|
||||||
|
sphinxcontrib-restbuilder
|
||||||
|
|||||||
@@ -292,6 +292,147 @@ Your URI template can use the following variables:
|
|||||||
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
|
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
|
||||||
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
|
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
|
||||||
|
|
||||||
|
.. tip::
|
||||||
|
Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
|
||||||
|
See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
|
||||||
|
Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
|
||||||
|
|
||||||
|
.. _cmake_workflow:
|
||||||
|
|
||||||
|
CMake Workflow
|
||||||
|
^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
|
||||||
|
This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
|
||||||
|
|
||||||
|
Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
|
||||||
|
Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
|
||||||
|
|
||||||
|
Key Files
|
||||||
|
~~~~~~~~~
|
||||||
|
|
||||||
|
These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
|
||||||
|
|
||||||
|
.. code-block:: text
|
||||||
|
|
||||||
|
.
|
||||||
|
├── .readthedocs.yml
|
||||||
|
├── CMakeLists.txt
|
||||||
|
├── CMakePresets.json
|
||||||
|
└── docs/
|
||||||
|
└── CMakeLists.txt
|
||||||
|
|
||||||
|
Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
|
||||||
|
|
||||||
|
.. dropdown:: .readthedocs.yml
|
||||||
|
:chevron: down-up
|
||||||
|
|
||||||
|
A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../.readthedocs.yml
|
||||||
|
:language: yaml
|
||||||
|
:lines: 1-9,11,14-
|
||||||
|
:linenos:
|
||||||
|
:emphasize-lines: 9, 14-15
|
||||||
|
|
||||||
|
.. list-table::
|
||||||
|
:header-rows: 1
|
||||||
|
:width: 100%
|
||||||
|
:widths: 15 85
|
||||||
|
|
||||||
|
* - Line
|
||||||
|
- Description
|
||||||
|
* - **9**
|
||||||
|
- Update the path if your requirements file is in a different location
|
||||||
|
* - **13-14**
|
||||||
|
- Modify the copy commands for the output formats you deploy
|
||||||
|
|
||||||
|
.. dropdown:: CMakeLists.txt
|
||||||
|
:chevron: down-up
|
||||||
|
|
||||||
|
A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../CMakeLists.txt
|
||||||
|
:language: cmake
|
||||||
|
:linenos:
|
||||||
|
:emphasize-lines: 9, 15
|
||||||
|
|
||||||
|
.. list-table::
|
||||||
|
:header-rows: 1
|
||||||
|
:width: 100%
|
||||||
|
:widths: 15 85
|
||||||
|
|
||||||
|
* - Line
|
||||||
|
- Description
|
||||||
|
* - **9**
|
||||||
|
- Update the ``GIT_TAG`` to use a different version or commit hash
|
||||||
|
* - **15**
|
||||||
|
- Change if your docs subdirectory has a different location
|
||||||
|
|
||||||
|
.. dropdown:: docs/CMakeLists.txt
|
||||||
|
:chevron: down-up
|
||||||
|
|
||||||
|
A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
|
||||||
|
|
||||||
|
.. literalinclude:: ../CMakeLists.txt
|
||||||
|
:language: cmake
|
||||||
|
:linenos:
|
||||||
|
:emphasize-lines: 5-7
|
||||||
|
|
||||||
|
.. list-table::
|
||||||
|
:header-rows: 1
|
||||||
|
:width: 100%
|
||||||
|
:widths: 15 85
|
||||||
|
|
||||||
|
* - Line
|
||||||
|
- Description
|
||||||
|
* - **5-7**
|
||||||
|
- Add or remove calls based on which output formats you need
|
||||||
|
|
||||||
|
|
||||||
|
.. dropdown:: CMakePresets.json
|
||||||
|
:chevron: down-up
|
||||||
|
|
||||||
|
Defines presets for configuring and building documentation:
|
||||||
|
|
||||||
|
- **Configure Presets:** Sets up the build directory.
|
||||||
|
- **Build Presets:** Defines build formats individually and all in parallel.
|
||||||
|
- **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
|
||||||
|
|
||||||
|
.. literalinclude:: ../../CMakePresets.json
|
||||||
|
:language: json
|
||||||
|
:linenos:
|
||||||
|
:emphasize-lines: 18-23, 24-29, 34
|
||||||
|
|
||||||
|
.. list-table::
|
||||||
|
:header-rows: 1
|
||||||
|
:width: 100%
|
||||||
|
:widths: 15 85
|
||||||
|
|
||||||
|
* - Line
|
||||||
|
- Description
|
||||||
|
* - **18-23**
|
||||||
|
- Remove this preset to disable Markdown documentation builds
|
||||||
|
* - **24-29**
|
||||||
|
- Remove this preset to disable reStructuredText documentation builds
|
||||||
|
* - **34**
|
||||||
|
- Modify the targets list to build only the output formats you need in parallel
|
||||||
|
|
||||||
|
Usage
|
||||||
|
~~~~~
|
||||||
|
|
||||||
|
To build documentation locally using CMake:
|
||||||
|
|
||||||
|
.. code-block:: console
|
||||||
|
|
||||||
|
# Run the full workflow (configure + build all formats)
|
||||||
|
cmake --workflow --preset documentation-workflow
|
||||||
|
|
||||||
|
# Or configure and build separately
|
||||||
|
cmake --preset documentation
|
||||||
|
cmake --build --preset html # Build HTML only
|
||||||
|
cmake --build --preset docs-parallel # Build all formats
|
||||||
|
|
||||||
.. _integration_examples:
|
.. _integration_examples:
|
||||||
|
|
||||||
Integration Examples
|
Integration Examples
|
||||||
|
|||||||
+9
-1
@@ -15,13 +15,17 @@ import subprocess
|
|||||||
project = "sphinx-llms-txt"
|
project = "sphinx-llms-txt"
|
||||||
copyright = "Jared Dillard"
|
copyright = "Jared Dillard"
|
||||||
author = "Jared Dillard"
|
author = "Jared Dillard"
|
||||||
llms_txt_uri_template = "{base_url}{docname}.md"
|
|
||||||
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
|
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
|
||||||
llms_txt_summary = """
|
llms_txt_summary = """
|
||||||
A Sphinx extension that generates a summary llms.txt file,written in Markdown,
|
A Sphinx extension that generates a summary llms.txt file,written in Markdown,
|
||||||
and a single combined documentation llms-full.txt file, written in reStructuredText.
|
and a single combined documentation llms-full.txt file, written in reStructuredText.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# This doesn't seem to be supported
|
||||||
|
# rst_file_suffix = ".html.rst"
|
||||||
|
markdown_file_suffix = ".html.md"
|
||||||
|
|
||||||
# check if the current commit is tagged as a release (vX.Y.Z)
|
# check if the current commit is tagged as a release (vX.Y.Z)
|
||||||
try:
|
try:
|
||||||
GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
|
GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
|
||||||
@@ -50,6 +54,10 @@ extensions = [
|
|||||||
"sphinx.ext.intersphinx",
|
"sphinx.ext.intersphinx",
|
||||||
"sphinx_contributors",
|
"sphinx_contributors",
|
||||||
"sphinx_llms_txt",
|
"sphinx_llms_txt",
|
||||||
|
"sphinxcontrib.restbuilder",
|
||||||
|
"sphinx_inline_tabs",
|
||||||
|
"sphinx.ext.extlinks",
|
||||||
|
"sphinx_design",
|
||||||
]
|
]
|
||||||
|
|
||||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||||
|
|||||||
@@ -4,15 +4,17 @@ Getting Started
|
|||||||
Installation
|
Installation
|
||||||
------------
|
------------
|
||||||
|
|
||||||
Directly install via ``pip`` by using:
|
Directly install by using:
|
||||||
|
|
||||||
|
.. tab:: via pip
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
pip install sphinx-llms-txt
|
pip install sphinx-llms-txt
|
||||||
|
|
||||||
Or with ``conda`` via ``conda-forge``:
|
.. tab:: via conda:
|
||||||
|
|
||||||
.. code::
|
.. code-block:: bash
|
||||||
|
|
||||||
conda install -c conda-forge sphinx-llms-txt
|
conda install -c conda-forge sphinx-llms-txt
|
||||||
|
|
||||||
@@ -32,7 +34,59 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
|
|||||||
sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
|
sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
|
||||||
sphinx-llms-txt: created /path/to/_build/html/llms.txt
|
sphinx-llms-txt: created /path/to/_build/html/llms.txt
|
||||||
|
|
||||||
|
.. _choosing-output-format:
|
||||||
|
|
||||||
.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades.
|
Choosing an Output Format
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
|
||||||
|
For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
|
||||||
|
|
||||||
|
.. list-table:: Output Format Comparison
|
||||||
|
:header-rows: 1
|
||||||
|
:widths: 18 27 27 27
|
||||||
|
|
||||||
|
* -
|
||||||
|
- Default
|
||||||
|
- Markdown
|
||||||
|
- reStructuredText
|
||||||
|
* - **Setup**
|
||||||
|
- No config
|
||||||
|
- CMake [#sphinxllm]_
|
||||||
|
- CMake
|
||||||
|
* - **Builder**
|
||||||
|
- Native [#native]_
|
||||||
|
- `sphinx-markdown-builder`_
|
||||||
|
- `sphinxcontrib-restbuilder`_
|
||||||
|
* - **Format**
|
||||||
|
- Raw reStructuredText source
|
||||||
|
- Rendered Markdown [#rendered]_
|
||||||
|
- Rendered reStructuredText [#rendered]_
|
||||||
|
* - **LLM Readability**
|
||||||
|
- Good - preserves structure for simple syntax
|
||||||
|
- Excellent - native LLM format
|
||||||
|
- Good - Can provide more structured content
|
||||||
|
* - **Key Advantage**
|
||||||
|
- Zero setup required
|
||||||
|
- More compact (less input tokens)
|
||||||
|
- Can preserve Sphinx semantics
|
||||||
|
* - **Key Disadvantage**
|
||||||
|
- Raw directives won't be parsed [#autodoc]_
|
||||||
|
- Loses structure from complex directives
|
||||||
|
- Can lose structure from complex directives
|
||||||
|
* - **llms-full.txt support**
|
||||||
|
- Supported with above caveats
|
||||||
|
- Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
|
||||||
|
- Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
|
||||||
|
|
||||||
|
.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
|
||||||
|
.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
|
||||||
|
|
||||||
|
.. rubric:: Footnotes
|
||||||
|
|
||||||
|
.. [#sphinxllm] See `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
|
||||||
|
.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
|
||||||
|
.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
|
||||||
|
.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
|
||||||
|
.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
|
||||||
|
|
||||||
See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.
|
|
||||||
|
|||||||
+18
-9
@@ -8,21 +8,28 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
|
|||||||
Demo
|
Demo
|
||||||
----
|
----
|
||||||
|
|
||||||
You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example.
|
This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
|
||||||
|
|
||||||
|
Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
|
||||||
|
|
||||||
Highlights
|
Highlights
|
||||||
----------
|
----------
|
||||||
|
|
||||||
1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build
|
**Zero Configuration**
|
||||||
2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content
|
Add the extension to your ``conf.py`` and you're done.
|
||||||
3. **Path Resolution**: Transforms relative paths in directives to full paths
|
The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
|
||||||
4. **Output Generation**: Creates two optional files:
|
|
||||||
|
|
||||||
- ``llms.txt``: A concise summary of your documentation, in Markdown
|
**Intelligent Content Processing**
|
||||||
- ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText
|
Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
|
||||||
|
|
||||||
5. **Content Filtering**: Allows you to exclude specific pages or sections
|
**Customizable When Needed**
|
||||||
6. **Source Code**: Allows you to include specific source code files
|
Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
|
||||||
|
See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
|
||||||
|
|
||||||
|
.. seealso::
|
||||||
|
|
||||||
|
For better default output without configuration, see `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ from NVIDIA.
|
||||||
|
sphinx-llms-txt is best when customized with alternative output formats, content filtering, or source code inclusion.
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 2
|
:maxdepth: 2
|
||||||
@@ -36,6 +43,8 @@ Highlights
|
|||||||
|
|
||||||
.. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
|
.. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
|
||||||
.. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
|
.. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
|
||||||
|
.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
|
||||||
|
.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
|
||||||
.. _Sphinx: http://sphinx-doc.org/
|
.. _Sphinx: http://sphinx-doc.org/
|
||||||
|
|
||||||
.. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg
|
.. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
|
|||||||
from .processor import DocumentProcessor
|
from .processor import DocumentProcessor
|
||||||
from .writer import FileWriter
|
from .writer import FileWriter
|
||||||
|
|
||||||
__version__ = "0.7.0"
|
__version__ = "0.7.1"
|
||||||
|
|
||||||
# Export classes needed by tests
|
# Export classes needed by tests
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -128,8 +128,11 @@ class DocumentProcessor:
|
|||||||
Returns:
|
Returns:
|
||||||
Processed content with directive paths properly resolved
|
Processed content with directive paths properly resolved
|
||||||
"""
|
"""
|
||||||
|
# Get code block ranges to skip directives inside them
|
||||||
|
code_block_ranges = self._get_code_block_ranges(content)
|
||||||
|
|
||||||
# Get the configured path directives to process
|
# Get the configured path directives to process
|
||||||
default_path_directives = ["image", "figure"]
|
default_path_directives = ["image", "figure", "literalinclude"]
|
||||||
custom_path_directives = self.config.get("llms_txt_directives")
|
custom_path_directives = self.config.get("llms_txt_directives")
|
||||||
path_directives = set(default_path_directives + custom_path_directives)
|
path_directives = set(default_path_directives + custom_path_directives)
|
||||||
|
|
||||||
@@ -143,6 +146,11 @@ class DocumentProcessor:
|
|||||||
is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
|
is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
|
||||||
|
|
||||||
def replace_directive_path(match, base_url=base_url, is_test=is_test):
|
def replace_directive_path(match, base_url=base_url, is_test=is_test):
|
||||||
|
# Check if this directive is within a code block
|
||||||
|
if self._is_in_code_block(match.start(), code_block_ranges):
|
||||||
|
# This directive is inside a code block, don't process it
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
prefix = match.group(1) # The entire directive prefix including whitespace
|
prefix = match.group(1) # The entire directive prefix including whitespace
|
||||||
path = match.group(3).strip() # The path argument
|
path = match.group(3).strip() # The path argument
|
||||||
|
|
||||||
@@ -276,6 +284,71 @@ class DocumentProcessor:
|
|||||||
|
|
||||||
return possible_paths
|
return possible_paths
|
||||||
|
|
||||||
|
def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
|
||||||
|
"""Find all code block ranges in the content.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content: The source content to analyze
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (start, end) tuples representing code block character
|
||||||
|
ranges
|
||||||
|
"""
|
||||||
|
code_block_ranges = []
|
||||||
|
|
||||||
|
# Match code block as well as `code` and `sourcecode` aliases
|
||||||
|
code_block_pattern = re.compile(
|
||||||
|
r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
|
||||||
|
)
|
||||||
|
|
||||||
|
for match in code_block_pattern.finditer(content):
|
||||||
|
start_pos = match.start()
|
||||||
|
indent = match.group(1)
|
||||||
|
indent_len = len(indent)
|
||||||
|
|
||||||
|
# Find the end of the code block by looking for the next line
|
||||||
|
# that is not indented more than the directive
|
||||||
|
block_start = match.end()
|
||||||
|
pos = block_start
|
||||||
|
|
||||||
|
# Skip any blank lines immediately after the directive
|
||||||
|
while pos < len(content) and content[pos] in "\n":
|
||||||
|
pos += 1
|
||||||
|
|
||||||
|
# Find where the code block ends
|
||||||
|
lines = content[pos:].split("\n")
|
||||||
|
block_end = pos
|
||||||
|
for line in lines:
|
||||||
|
if line.strip(): # Non-empty line
|
||||||
|
# Check indentation level
|
||||||
|
line_indent = len(line) - len(line.lstrip())
|
||||||
|
if line_indent <= indent_len:
|
||||||
|
# The block ends when we find a line that is indented
|
||||||
|
# less than the directive itself
|
||||||
|
break
|
||||||
|
block_end += len(line) + 1 # +1 for the newline
|
||||||
|
|
||||||
|
code_block_ranges.append((start_pos, block_end))
|
||||||
|
|
||||||
|
return code_block_ranges
|
||||||
|
|
||||||
|
def _is_in_code_block(
|
||||||
|
self, match_start: int, code_block_ranges: List[Tuple[int, int]]
|
||||||
|
) -> bool:
|
||||||
|
"""Check if a match position is within a code block.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
match_start: The starting position of the match
|
||||||
|
code_block_ranges: List of (start, end) tuples for code blocks
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the match is within a code block, False otherwise
|
||||||
|
"""
|
||||||
|
for block_start, block_end in code_block_ranges:
|
||||||
|
if block_start <= match_start < block_end:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _process_includes(self, content: str, source_path: Path) -> str:
|
def _process_includes(self, content: str, source_path: Path) -> str:
|
||||||
"""Process include directives in content.
|
"""Process include directives in content.
|
||||||
|
|
||||||
@@ -286,11 +359,18 @@ class DocumentProcessor:
|
|||||||
Returns:
|
Returns:
|
||||||
Processed content with include directives replaced with included content
|
Processed content with include directives replaced with included content
|
||||||
"""
|
"""
|
||||||
|
code_block_ranges = self._get_code_block_ranges(content)
|
||||||
|
|
||||||
# Find all include directives using regex
|
# Find all include directives using regex
|
||||||
include_pattern = build_directive_pattern(["include"])
|
include_pattern = build_directive_pattern(["include"])
|
||||||
|
|
||||||
# Function to replace each include with content
|
# Function to replace each include with content
|
||||||
def replace_include(match):
|
def replace_include(match):
|
||||||
|
# Check if this include is within a code block
|
||||||
|
if self._is_in_code_block(match.start(), code_block_ranges):
|
||||||
|
# This include is inside a code block, don't process it
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
include_path = match.group(3)
|
include_path = match.group(3)
|
||||||
directive_part = match.group(
|
directive_part = match.group(
|
||||||
1
|
1
|
||||||
|
|||||||
@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
|
|||||||
assert processed_content == expected_content
|
assert processed_content == expected_content
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_includes_in_code_block(tmp_path):
|
||||||
|
"""Test that an `include` within a `code-block` is not processed."""
|
||||||
|
# Create a processor
|
||||||
|
config = {"llms_txt_directives": []}
|
||||||
|
processor = DocumentProcessor(config)
|
||||||
|
|
||||||
|
# Create a source file that uses include syntax within a `code-block`
|
||||||
|
source_content = (
|
||||||
|
"Normal paragraph.\n\n"
|
||||||
|
".. code-block:: rst\n\n"
|
||||||
|
" .. include:: foo.txt\n\n"
|
||||||
|
"Another normal paragraph."
|
||||||
|
)
|
||||||
|
source_file = tmp_path / "source.txt"
|
||||||
|
with open(source_file, "w", encoding="utf-8") as f:
|
||||||
|
f.write(source_content)
|
||||||
|
|
||||||
|
# Run the include directive processor
|
||||||
|
processed_content = processor._process_includes(source_content, source_file)
|
||||||
|
|
||||||
|
# Check that the include directive was not processed
|
||||||
|
expected_content = source_content
|
||||||
|
assert processed_content == expected_content
|
||||||
|
|
||||||
|
|
||||||
def test_process_includes_with_relative_paths(tmp_path):
|
def test_process_includes_with_relative_paths(tmp_path):
|
||||||
"""Test that include directives with relative paths are processed correctly."""
|
"""Test that include directives with relative paths are processed correctly."""
|
||||||
# Create a processor
|
# Create a processor
|
||||||
|
|||||||
Reference in New Issue
Block a user