Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5bfbb0168f | ||
|
|
e2a80faf04 | ||
|
|
b63801bcff | ||
|
|
c45ebb0369 |
@@ -10,9 +10,9 @@ jobs:
|
|||||||
pre-commit:
|
pre-commit:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v5
|
||||||
- name: Set up Python 3.10
|
- name: Set up Python 3.10
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: "3.10"
|
python-version: "3.10"
|
||||||
- uses: pre-commit/action@v3.0.1
|
- uses: pre-commit/action@v3.0.1
|
||||||
@@ -23,10 +23,10 @@ jobs:
|
|||||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v5
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v5
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
|
|||||||
+10
-11
@@ -1,15 +1,14 @@
|
|||||||
version: 2
|
version: 2
|
||||||
|
|
||||||
build:
|
build:
|
||||||
os: "ubuntu-20.04"
|
os: ubuntu-24.04
|
||||||
tools:
|
tools:
|
||||||
python: "3.10"
|
python: "3.13"
|
||||||
|
commands:
|
||||||
sphinx:
|
- pip install cmake
|
||||||
configuration: docs/source/conf.py
|
- pip install -r docs/requirements.txt
|
||||||
|
- pip install -e .
|
||||||
python:
|
- cmake --workflow --preset documentation-workflow
|
||||||
install:
|
# Copy built documentation to Read the Docs output directory
|
||||||
- requirements: docs/requirements.txt
|
- mkdir -p $READTHEDOCS_OUTPUT/html
|
||||||
- method: pip
|
- cp -r build/* $READTHEDOCS_OUTPUT/html/
|
||||||
path: .
|
|
||||||
|
|||||||
@@ -1,6 +1,18 @@
|
|||||||
Changelog
|
Changelog
|
||||||
=========
|
=========
|
||||||
|
|
||||||
|
0.7.0
|
||||||
|
-----
|
||||||
|
|
||||||
|
- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`.
|
||||||
|
`#48 <https://github.com/jdillard/sphinx-llms-txt/pull/48>`_
|
||||||
|
|
||||||
|
0.6.0
|
||||||
|
-----
|
||||||
|
|
||||||
|
- Improve _sources directory handling
|
||||||
|
`#47 <https://github.com/jdillard/sphinx-llms-txt/pull/47>`_
|
||||||
|
|
||||||
0.5.3
|
0.5.3
|
||||||
-----
|
-----
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,8 @@
|
|||||||
|
cmake_minimum_required(VERSION 3.15)
|
||||||
|
project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
|
||||||
|
|
||||||
|
# Add CMake module path
|
||||||
|
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||||
|
|
||||||
|
# Add documentation
|
||||||
|
add_subdirectory(docs)
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"version": 6,
|
||||||
|
"configurePresets": [
|
||||||
|
{
|
||||||
|
"name": "documentation",
|
||||||
|
"displayName": "Documentation Build",
|
||||||
|
"description": "Configure project with documentation environment setup",
|
||||||
|
"binaryDir": "${sourceDir}/build"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"buildPresets": [
|
||||||
|
{
|
||||||
|
"name": "html",
|
||||||
|
"displayName": "Build HTML Documentation",
|
||||||
|
"configurePreset": "documentation",
|
||||||
|
"targets": ["html"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "markdown",
|
||||||
|
"displayName": "Build Markdown Documentation",
|
||||||
|
"configurePreset": "documentation",
|
||||||
|
"targets": ["markdown"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "docs-parallel",
|
||||||
|
"displayName": "Build HTML and Markdown in parallel",
|
||||||
|
"configurePreset": "documentation",
|
||||||
|
"targets": ["html", "markdown"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"workflowPresets": [
|
||||||
|
{
|
||||||
|
"name": "documentation-workflow",
|
||||||
|
"displayName": "Documentation Build Workflow",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"type": "configure",
|
||||||
|
"name": "documentation"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "build",
|
||||||
|
"name": "docs-parallel"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# SetupSphinx.cmake - Find Sphinx for CI environments
|
||||||
|
|
||||||
|
# Function to find Sphinx in the system
|
||||||
|
function(setup_sphinx_environment)
|
||||||
|
# Find sphinx-build executable in system
|
||||||
|
find_program(SPHINX_EXECUTABLE
|
||||||
|
NAMES sphinx-build
|
||||||
|
DOC "Sphinx documentation generator"
|
||||||
|
)
|
||||||
|
|
||||||
|
if(NOT SPHINX_EXECUTABLE)
|
||||||
|
message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# Get Sphinx version
|
||||||
|
execute_process(
|
||||||
|
COMMAND ${SPHINX_EXECUTABLE} --version
|
||||||
|
OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
|
||||||
|
ERROR_VARIABLE SPHINX_VERSION_OUTPUT
|
||||||
|
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||||
|
)
|
||||||
|
message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
|
||||||
|
message(STATUS "${SPHINX_VERSION_OUTPUT}")
|
||||||
|
|
||||||
|
# Export to parent scope
|
||||||
|
set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
|
||||||
|
endfunction()
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Find Sphinx
|
||||||
|
include(SetupSphinx)
|
||||||
|
setup_sphinx_environment()
|
||||||
|
|
||||||
|
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
|
||||||
|
set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
|
||||||
|
|
||||||
|
# Build HTML documentation
|
||||||
|
add_custom_target(html
|
||||||
|
COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
||||||
|
COMMENT "Building HTML documentation"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
|
||||||
|
# Build markdown documentation
|
||||||
|
add_custom_target(markdown
|
||||||
|
COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
||||||
|
COMMENT "Building Markdown documentation"
|
||||||
|
VERBATIM
|
||||||
|
)
|
||||||
|
|
||||||
|
# Build both formats
|
||||||
|
add_custom_target(docs ALL)
|
||||||
|
add_dependencies(docs html markdown)
|
||||||
@@ -4,3 +4,4 @@ sphinx-contributors
|
|||||||
sphinx
|
sphinx
|
||||||
sphinx-llms-txt
|
sphinx-llms-txt
|
||||||
sphinxext-opengraph
|
sphinxext-opengraph
|
||||||
|
sphinx-markdown-builder
|
||||||
|
|||||||
@@ -261,6 +261,37 @@ If you want to include absolute URLs for resources in your documentation, you ca
|
|||||||
|
|
||||||
When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
|
When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
|
||||||
|
|
||||||
|
.. _customizing_uri_links:
|
||||||
|
|
||||||
|
Customizing URI Links in llms.txt
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available.
|
||||||
|
You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
# Default: Link to source files, if _sources exists
|
||||||
|
llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||||
|
|
||||||
|
# Default: Link to HTML pages instead, if _sources doesn't exist
|
||||||
|
llms_txt_uri_template = "{base_url}{docname}.html"
|
||||||
|
|
||||||
|
# Manual: Link to a custom markdown build
|
||||||
|
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||||
|
|
||||||
|
.. _available_template_variables:
|
||||||
|
|
||||||
|
Available Template Variables
|
||||||
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
Your URI template can use the following variables:
|
||||||
|
|
||||||
|
- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash)
|
||||||
|
- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``)
|
||||||
|
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
|
||||||
|
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
|
||||||
|
|
||||||
.. _integration_examples:
|
.. _integration_examples:
|
||||||
|
|
||||||
Integration Examples
|
Integration Examples
|
||||||
@@ -285,6 +316,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
|
|||||||
This is a comprehensive documentation set for our project.
|
This is a comprehensive documentation set for our project.
|
||||||
It includes API references, usage examples, and tutorials.
|
It includes API references, usage examples, and tutorials.
|
||||||
"""
|
"""
|
||||||
|
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||||
|
|
||||||
# Path handling
|
# Path handling
|
||||||
html_baseurl = "https://docs.example.com/"
|
html_baseurl = "https://docs.example.com/"
|
||||||
|
|||||||
+2
-1
@@ -15,6 +15,7 @@ import subprocess
|
|||||||
project = "sphinx-llms-txt"
|
project = "sphinx-llms-txt"
|
||||||
copyright = "Jared Dillard"
|
copyright = "Jared Dillard"
|
||||||
author = "Jared Dillard"
|
author = "Jared Dillard"
|
||||||
|
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||||
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
|
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
|
||||||
llms_txt_summary = """
|
llms_txt_summary = """
|
||||||
A Sphinx extension that generates a summary llms.txt file,written in Markdown,
|
A Sphinx extension that generates a summary llms.txt file,written in Markdown,
|
||||||
@@ -88,7 +89,7 @@ html_theme_options = {
|
|||||||
"source_directory": "docs/source/",
|
"source_directory": "docs/source/",
|
||||||
}
|
}
|
||||||
|
|
||||||
html_baseurl = "https://sphinx-llms-txt.readthedocs.org/"
|
html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/"
|
||||||
|
|
||||||
|
|
||||||
# -- Options for HTMLHelp output ---------------------------------------------
|
# -- Options for HTMLHelp output ---------------------------------------------
|
||||||
|
|||||||
@@ -58,6 +58,15 @@ Project Configuration Values
|
|||||||
|
|
||||||
.. versionadded:: 0.2.0
|
.. versionadded:: 0.2.0
|
||||||
|
|
||||||
|
.. confval:: llms_txt_uri_template
|
||||||
|
|
||||||
|
- **Type**: string or ``None``
|
||||||
|
- **Default**: ``None``
|
||||||
|
- **Description**: Template string for generating URIs in ``llms.txt``.
|
||||||
|
See :ref:`customizing_uri_links`.
|
||||||
|
|
||||||
|
.. versionadded:: 0.7.0
|
||||||
|
|
||||||
.. confval:: llms_txt_directives
|
.. confval:: llms_txt_directives
|
||||||
|
|
||||||
- **Type**: list of strings
|
- **Type**: list of strings
|
||||||
|
|||||||
@@ -48,6 +48,8 @@ test = [
|
|||||||
"pytest>=7.0.0",
|
"pytest>=7.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[tool.setuptools]
|
||||||
|
packages = ["sphinx_llms_txt"]
|
||||||
|
|
||||||
[tool.setuptools.dynamic]
|
[tool.setuptools.dynamic]
|
||||||
version = {attr = "sphinx_llms_txt.__version__"}
|
version = {attr = "sphinx_llms_txt.__version__"}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
|
|||||||
from .processor import DocumentProcessor
|
from .processor import DocumentProcessor
|
||||||
from .writer import FileWriter
|
from .writer import FileWriter
|
||||||
|
|
||||||
__version__ = "0.5.3"
|
__version__ = "0.7.0"
|
||||||
|
|
||||||
# Export classes needed by tests
|
# Export classes needed by tests
|
||||||
__all__ = [
|
__all__ = [
|
||||||
@@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception):
|
|||||||
config = {
|
config = {
|
||||||
"llms_txt_file": app.config.llms_txt_file,
|
"llms_txt_file": app.config.llms_txt_file,
|
||||||
"llms_txt_filename": app.config.llms_txt_filename,
|
"llms_txt_filename": app.config.llms_txt_filename,
|
||||||
|
"llms_txt_uri_template": app.config.llms_txt_uri_template,
|
||||||
"llms_txt_title": app.config.llms_txt_title,
|
"llms_txt_title": app.config.llms_txt_title,
|
||||||
"llms_txt_summary": summary,
|
"llms_txt_summary": summary,
|
||||||
"llms_txt_full_file": app.config.llms_txt_full_file,
|
"llms_txt_full_file": app.config.llms_txt_full_file,
|
||||||
@@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
|
|||||||
|
|
||||||
app.add_config_value("llms_txt_file", True, "env")
|
app.add_config_value("llms_txt_file", True, "env")
|
||||||
app.add_config_value("llms_txt_filename", "llms.txt", "env")
|
app.add_config_value("llms_txt_filename", "llms.txt", "env")
|
||||||
|
app.add_config_value("llms_txt_uri_template", None, "env")
|
||||||
app.add_config_value("llms_txt_full_file", True, "env")
|
app.add_config_value("llms_txt_full_file", True, "env")
|
||||||
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
|
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
|
||||||
app.add_config_value("llms_txt_full_max_size", None, "env")
|
app.add_config_value("llms_txt_full_max_size", None, "env")
|
||||||
|
|||||||
+35
-12
@@ -204,25 +204,43 @@ class LLMSFullManager:
|
|||||||
sources_dir = path
|
sources_dir = path
|
||||||
break
|
break
|
||||||
|
|
||||||
if not sources_dir:
|
# Get the correct page order (with or without source suffixes)
|
||||||
logger.warning(
|
|
||||||
"Could not find _sources directory, skipping llms-full creation"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
|
|
||||||
# Get the correct page order with source suffixes
|
|
||||||
page_order = self.collector.get_page_order(sources_dir)
|
page_order = self.collector.get_page_order(sources_dir)
|
||||||
|
|
||||||
if not page_order:
|
if not page_order:
|
||||||
logger.warning(
|
logger.warning("Could not determine page order, skipping file generation")
|
||||||
"Could not determine page order, skipping llms-full creation"
|
|
||||||
)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
# Apply exclusion filter if configured
|
# Apply exclusion filter if configured
|
||||||
page_order = self.collector.filter_excluded_pages(page_order)
|
page_order = self.collector.filter_excluded_pages(page_order)
|
||||||
|
|
||||||
# Determine output file name and location
|
# If no sources directory, only generate llms.txt and return early
|
||||||
|
if not sources_dir:
|
||||||
|
# Generate llms.txt if requested
|
||||||
|
if self.config.get("llms_txt_file"):
|
||||||
|
filtered_page_order = self._filter_ignored_pages(page_order)
|
||||||
|
self.writer.write_verbose_info_to_file(
|
||||||
|
filtered_page_order,
|
||||||
|
self.collector.page_titles,
|
||||||
|
0, # No line count since no llms-full.txt
|
||||||
|
sources_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Only warn if user explicitly wants llms-full.txt
|
||||||
|
if self.config.get("llms_txt_full_file"):
|
||||||
|
# Check if html_copy_source is False
|
||||||
|
if self.app and not self.app.config.html_copy_source:
|
||||||
|
logger.warning(
|
||||||
|
"Could not find _sources directory, skipping llms-full.txt."
|
||||||
|
"Set html_copy_source = True in conf.py to enable."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Could not find _sources directory, skipping llms-full.txt"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Determine output file name and location for llms-full.txt
|
||||||
output_filename = self.config.get("llms_txt_full_filename")
|
output_filename = self.config.get("llms_txt_full_filename")
|
||||||
output_path = Path(outdir) / output_filename
|
output_path = Path(outdir) / output_filename
|
||||||
|
|
||||||
@@ -506,6 +524,7 @@ class LLMSFullManager:
|
|||||||
filtered_page_order,
|
filtered_page_order,
|
||||||
self.collector.page_titles,
|
self.collector.page_titles,
|
||||||
total_line_count,
|
total_line_count,
|
||||||
|
sources_dir,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
elif action == "note":
|
elif action == "note":
|
||||||
@@ -519,6 +538,7 @@ class LLMSFullManager:
|
|||||||
filtered_page_order,
|
filtered_page_order,
|
||||||
self.collector.page_titles,
|
self.collector.page_titles,
|
||||||
total_line_count,
|
total_line_count,
|
||||||
|
sources_dir,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
elif action == "keep":
|
elif action == "keep":
|
||||||
@@ -537,7 +557,10 @@ class LLMSFullManager:
|
|||||||
if success and self.config.get("llms_txt_file"):
|
if success and self.config.get("llms_txt_file"):
|
||||||
filtered_page_order = self._filter_ignored_pages(page_order)
|
filtered_page_order = self._filter_ignored_pages(page_order)
|
||||||
self.writer.write_verbose_info_to_file(
|
self.writer.write_verbose_info_to_file(
|
||||||
filtered_page_order, self.collector.page_titles, total_line_count
|
filtered_page_order,
|
||||||
|
self.collector.page_titles,
|
||||||
|
total_line_count,
|
||||||
|
sources_dir,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:
|
def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:
|
||||||
|
|||||||
@@ -19,6 +19,42 @@ class FileWriter:
|
|||||||
self.outdir = outdir
|
self.outdir = outdir
|
||||||
self.app = app
|
self.app = app
|
||||||
|
|
||||||
|
def _resolve_uri_template(self, sources_dir: Path = None) -> str:
|
||||||
|
"""Resolve which URI template to use based on configuration and sources_dir.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sources_dir: Path to _sources directory (None if not found)
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The template string to use for generating URIs
|
||||||
|
"""
|
||||||
|
# If custom template exists
|
||||||
|
custom_template = self.config.get("llms_txt_uri_template")
|
||||||
|
|
||||||
|
if custom_template:
|
||||||
|
# Validate user's template by checking for valid variable names
|
||||||
|
try:
|
||||||
|
# Try formatting with test valid values to validate syntax
|
||||||
|
test_values = {
|
||||||
|
"base_url": "http://example.com/",
|
||||||
|
"docname": "test",
|
||||||
|
"suffix": ".rst",
|
||||||
|
"sourcelink_suffix": ".txt",
|
||||||
|
}
|
||||||
|
custom_template.format(**test_values)
|
||||||
|
return custom_template
|
||||||
|
except (KeyError, ValueError) as e:
|
||||||
|
logger.warning(
|
||||||
|
f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. "
|
||||||
|
f"Falling back to default."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Else, use one of the default templates
|
||||||
|
if sources_dir:
|
||||||
|
return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||||
|
else:
|
||||||
|
return "{base_url}{docname}.html"
|
||||||
|
|
||||||
def write_combined_file(
|
def write_combined_file(
|
||||||
self, content_parts: List[str], output_path: Path, total_line_count: int
|
self, content_parts: List[str], output_path: Path, total_line_count: int
|
||||||
) -> bool:
|
) -> bool:
|
||||||
@@ -50,6 +86,7 @@ class FileWriter:
|
|||||||
page_order: Union[List[str], List[Tuple[str, str]]],
|
page_order: Union[List[str], List[Tuple[str, str]]],
|
||||||
page_titles: Dict[str, str],
|
page_titles: Dict[str, str],
|
||||||
total_line_count: int = 0,
|
total_line_count: int = 0,
|
||||||
|
sources_dir: Path = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Write summary information to the llms.txt file.
|
"""Write summary information to the llms.txt file.
|
||||||
|
|
||||||
@@ -57,6 +94,7 @@ class FileWriter:
|
|||||||
page_order: Ordered list of document names or (docname, suffix) tuples
|
page_order: Ordered list of document names or (docname, suffix) tuples
|
||||||
page_titles: Dictionary mapping docnames to titles
|
page_titles: Dictionary mapping docnames to titles
|
||||||
total_line_count: Total number of lines in the combined content
|
total_line_count: Total number of lines in the combined content
|
||||||
|
sources_dir: Path to _sources directory (None if not found)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
True if successful, False otherwise
|
True if successful, False otherwise
|
||||||
@@ -102,14 +140,37 @@ class FileWriter:
|
|||||||
if not base_url.endswith("/"):
|
if not base_url.endswith("/"):
|
||||||
base_url += "/"
|
base_url += "/"
|
||||||
|
|
||||||
|
# Get sourcelink suffix from Sphinx config
|
||||||
|
sourcelink_suffix = ""
|
||||||
|
if self.app and hasattr(self.app.config, "html_sourcelink_suffix"):
|
||||||
|
sourcelink_suffix = self.app.config.html_sourcelink_suffix
|
||||||
|
# Handle empty string case specially
|
||||||
|
if sourcelink_suffix == "":
|
||||||
|
sourcelink_suffix = "" # Keep it empty
|
||||||
|
elif not sourcelink_suffix.startswith("."):
|
||||||
|
sourcelink_suffix = "." + sourcelink_suffix
|
||||||
|
|
||||||
|
# Resolve which template to use
|
||||||
|
uri_template = self._resolve_uri_template(sources_dir)
|
||||||
|
|
||||||
for item in page_order:
|
for item in page_order:
|
||||||
# Handle both old format (str) and new format (tuple)
|
# Handle both old format (str) and new format (tuple)
|
||||||
if isinstance(item, tuple):
|
if isinstance(item, tuple):
|
||||||
docname, _ = item
|
docname, suffix = item
|
||||||
else:
|
else:
|
||||||
docname = item
|
docname = item
|
||||||
|
suffix = None
|
||||||
|
|
||||||
title = page_titles.get(docname, docname)
|
title = page_titles.get(docname, docname)
|
||||||
f.write(f"- [{title}]({base_url}{docname}.html)\n")
|
|
||||||
|
uri = uri_template.format(
|
||||||
|
base_url=base_url,
|
||||||
|
docname=docname,
|
||||||
|
suffix=suffix or "",
|
||||||
|
sourcelink_suffix=sourcelink_suffix,
|
||||||
|
)
|
||||||
|
|
||||||
|
f.write(f"- [{title}]({uri})\n")
|
||||||
|
|
||||||
logger.info(f"sphinx-llms-txt: created {output_path}")
|
logger.info(f"sphinx-llms-txt: created {output_path}")
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -794,6 +794,7 @@ def test_summary_default_uses_first_paragraph():
|
|||||||
llms_txt_summary = None # Not configured
|
llms_txt_summary = None # Not configured
|
||||||
llms_txt_file = True
|
llms_txt_file = True
|
||||||
llms_txt_filename = "llms.txt"
|
llms_txt_filename = "llms.txt"
|
||||||
|
llms_txt_uri_template = None
|
||||||
llms_txt_title = None
|
llms_txt_title = None
|
||||||
llms_txt_full_file = True
|
llms_txt_full_file = True
|
||||||
llms_txt_full_filename = "llms-full.txt"
|
llms_txt_full_filename = "llms-full.txt"
|
||||||
@@ -1031,3 +1032,128 @@ def test_code_files_ignored_patterns(tmp_path, caplog):
|
|||||||
assert (
|
assert (
|
||||||
"Code file pattern 'docs/**/*.rst' ignored." in captured_warnings[0]
|
"Code file pattern 'docs/**/*.rst' ignored." in captured_warnings[0]
|
||||||
), f"Warning message should contain expected text. Got: {captured_warnings[0]}"
|
), f"Warning message should contain expected text. Got: {captured_warnings[0]}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_llms_txt_generated_without_sources_dir(tmp_path):
|
||||||
|
"""Test that llms.txt is generated even when _sources directory doesn't exist."""
|
||||||
|
|
||||||
|
from sphinx_llms_txt.manager import LLMSFullManager
|
||||||
|
|
||||||
|
# Create manager
|
||||||
|
manager = LLMSFullManager()
|
||||||
|
|
||||||
|
# Set config to enable llms.txt
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
"llms_txt_full_file": True,
|
||||||
|
"llms_txt_full_filename": "llms-full.txt",
|
||||||
|
"llms_txt_exclude": [],
|
||||||
|
"llms_txt_directives": [],
|
||||||
|
}
|
||||||
|
manager.set_config(config)
|
||||||
|
|
||||||
|
# Create directories (but no _sources)
|
||||||
|
outdir = tmp_path / "build"
|
||||||
|
srcdir = tmp_path / "source"
|
||||||
|
outdir.mkdir()
|
||||||
|
srcdir.mkdir()
|
||||||
|
|
||||||
|
# Mock env with documents
|
||||||
|
class MockEnv:
|
||||||
|
all_docs = {"index": None, "about": None}
|
||||||
|
titles = {
|
||||||
|
"index": type("TitleNode", (), {"astext": lambda self: "Home"})(),
|
||||||
|
"about": type("TitleNode", (), {"astext": lambda self: "About"})(),
|
||||||
|
}
|
||||||
|
toctree_includes = {"index": ["about"]}
|
||||||
|
|
||||||
|
manager.set_env(MockEnv())
|
||||||
|
manager.set_master_doc("index")
|
||||||
|
|
||||||
|
# Update page titles directly in the collector
|
||||||
|
manager.update_page_title("index", "Home")
|
||||||
|
manager.update_page_title("about", "About")
|
||||||
|
|
||||||
|
# Call combine_sources - should generate llms.txt even without _sources
|
||||||
|
manager.combine_sources(str(outdir), str(srcdir))
|
||||||
|
|
||||||
|
# Verify llms.txt was created
|
||||||
|
llms_txt = outdir / "llms.txt"
|
||||||
|
assert llms_txt.exists(), "llms.txt should be generated even without _sources"
|
||||||
|
|
||||||
|
# Verify llms-full.txt was NOT created (since no _sources)
|
||||||
|
llms_full_txt = outdir / "llms-full.txt"
|
||||||
|
assert (
|
||||||
|
not llms_full_txt.exists()
|
||||||
|
), "llms-full.txt should not be generated without _sources"
|
||||||
|
|
||||||
|
# Read llms.txt and verify it has content
|
||||||
|
with open(llms_txt, "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Should contain page titles and links
|
||||||
|
assert "Home" in content
|
||||||
|
assert "About" in content
|
||||||
|
assert "index.html" in content
|
||||||
|
assert "about.html" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
|
||||||
|
"""
|
||||||
|
Test that no warning is logged when llms_txt_full_file=False and
|
||||||
|
_sources doesn't exist.
|
||||||
|
"""
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from sphinx_llms_txt.manager import LLMSFullManager
|
||||||
|
|
||||||
|
# Create manager
|
||||||
|
manager = LLMSFullManager()
|
||||||
|
|
||||||
|
# Set config with llms_txt_full_file=False
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
"llms_txt_full_file": False, # User doesn't want llms-full.txt
|
||||||
|
"llms_txt_full_filename": "llms-full.txt",
|
||||||
|
"llms_txt_exclude": [],
|
||||||
|
"llms_txt_directives": [],
|
||||||
|
}
|
||||||
|
manager.set_config(config)
|
||||||
|
|
||||||
|
# Create directories (but no _sources)
|
||||||
|
outdir = tmp_path / "build"
|
||||||
|
srcdir = tmp_path / "source"
|
||||||
|
outdir.mkdir()
|
||||||
|
srcdir.mkdir()
|
||||||
|
|
||||||
|
# Mock env with documents
|
||||||
|
class MockEnv:
|
||||||
|
all_docs = {"index": None}
|
||||||
|
titles = {"index": type("TitleNode", (), {"astext": lambda self: "Home"})()}
|
||||||
|
toctree_includes = {"index": []}
|
||||||
|
|
||||||
|
manager.set_env(MockEnv())
|
||||||
|
manager.set_master_doc("index")
|
||||||
|
manager.update_page_title("index", "Home")
|
||||||
|
|
||||||
|
# Capture warnings
|
||||||
|
captured_warnings = []
|
||||||
|
|
||||||
|
def capture_warning(message, *args, **kwargs):
|
||||||
|
if "_sources" in str(message):
|
||||||
|
captured_warnings.append(message)
|
||||||
|
|
||||||
|
with patch("sphinx_llms_txt.manager.logger.warning", side_effect=capture_warning):
|
||||||
|
# Call combine_sources
|
||||||
|
manager.combine_sources(str(outdir), str(srcdir))
|
||||||
|
|
||||||
|
# Verify NO warning was logged since llms_txt_full_file=False
|
||||||
|
assert (
|
||||||
|
len(captured_warnings) == 0
|
||||||
|
), "No warning should be logged when llms_txt_full_file=False"
|
||||||
|
|
||||||
|
# Verify llms.txt was still created
|
||||||
|
llms_txt = outdir / "llms.txt"
|
||||||
|
assert llms_txt.exists()
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
"""Test URI template functionality for llms.txt links."""
|
||||||
|
|
||||||
|
from sphinx_llms_txt import FileWriter
|
||||||
|
|
||||||
|
|
||||||
|
def test_uri_template_with_sources_dir(tmp_path):
|
||||||
|
"""Test that default template uses _sources links when sources_dir exists."""
|
||||||
|
build_dir = tmp_path / "build"
|
||||||
|
build_dir.mkdir()
|
||||||
|
|
||||||
|
# Create _sources directory to simulate its existence
|
||||||
|
sources_dir = build_dir / "_sources"
|
||||||
|
sources_dir.mkdir()
|
||||||
|
|
||||||
|
# Mock app with html_sourcelink_suffix
|
||||||
|
class MockApp:
|
||||||
|
class Config:
|
||||||
|
html_sourcelink_suffix = ".txt"
|
||||||
|
|
||||||
|
config = Config()
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
"llms_txt_uri_template": (
|
||||||
|
"{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||||
|
),
|
||||||
|
"html_baseurl": "https://example.com",
|
||||||
|
}
|
||||||
|
writer = FileWriter(config, str(build_dir), MockApp())
|
||||||
|
|
||||||
|
page_titles = {
|
||||||
|
"index": "Home Page",
|
||||||
|
"about": "About Us",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Page order with suffixes (simulating _sources files exist)
|
||||||
|
page_order = [("index", ".rst"), ("about", ".md")]
|
||||||
|
|
||||||
|
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||||
|
|
||||||
|
# Check that the file was created
|
||||||
|
verbose_file = build_dir / "llms.txt"
|
||||||
|
assert verbose_file.exists()
|
||||||
|
|
||||||
|
# Read the file content
|
||||||
|
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Should link to _sources files
|
||||||
|
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
|
||||||
|
assert "- [About Us](https://example.com/_sources/about.md.txt)" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_uri_template_without_sources_dir(tmp_path):
|
||||||
|
"""
|
||||||
|
Test that HTML template is used when sources_dir doesn't exist and no custom
|
||||||
|
template.
|
||||||
|
"""
|
||||||
|
build_dir = tmp_path / "build"
|
||||||
|
build_dir.mkdir()
|
||||||
|
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
# No custom template set
|
||||||
|
"html_baseurl": "https://example.com",
|
||||||
|
}
|
||||||
|
writer = FileWriter(config, str(build_dir))
|
||||||
|
|
||||||
|
page_titles = {
|
||||||
|
"index": "Home Page",
|
||||||
|
"about": "About Us",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Page order without suffixes (simulating no _sources)
|
||||||
|
page_order = [("index", None), ("about", None)]
|
||||||
|
|
||||||
|
# Pass None for sources_dir to simulate it doesn't exist
|
||||||
|
writer.write_verbose_info_to_file(page_order, page_titles, 0, None)
|
||||||
|
|
||||||
|
# Check that the file was created
|
||||||
|
verbose_file = build_dir / "llms.txt"
|
||||||
|
assert verbose_file.exists()
|
||||||
|
|
||||||
|
# Read the file content
|
||||||
|
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Should fallback to HTML links
|
||||||
|
assert "- [Home Page](https://example.com/index.html)" in content
|
||||||
|
assert "- [About Us](https://example.com/about.html)" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_uri_template_custom(tmp_path):
|
||||||
|
"""Test that custom URI template works correctly."""
|
||||||
|
build_dir = tmp_path / "build"
|
||||||
|
build_dir.mkdir()
|
||||||
|
|
||||||
|
sources_dir = build_dir / "_sources"
|
||||||
|
sources_dir.mkdir()
|
||||||
|
|
||||||
|
# Mock app with html_sourcelink_suffix
|
||||||
|
class MockApp:
|
||||||
|
class Config:
|
||||||
|
html_sourcelink_suffix = ".txt"
|
||||||
|
|
||||||
|
config = Config()
|
||||||
|
|
||||||
|
# Custom template that uses different path
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
"llms_txt_uri_template": "{base_url}raw/{docname}{suffix}",
|
||||||
|
"html_baseurl": "https://example.com/",
|
||||||
|
}
|
||||||
|
writer = FileWriter(config, str(build_dir), MockApp())
|
||||||
|
|
||||||
|
page_titles = {
|
||||||
|
"index": "Home Page",
|
||||||
|
}
|
||||||
|
|
||||||
|
page_order = [("index", ".rst")]
|
||||||
|
|
||||||
|
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||||
|
|
||||||
|
verbose_file = build_dir / "llms.txt"
|
||||||
|
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Should use custom template
|
||||||
|
assert "- [Home Page](https://example.com/raw/index.rst)" in content
|
||||||
|
|
||||||
|
|
||||||
|
def test_uri_template_invalid_fallback(tmp_path):
|
||||||
|
"""
|
||||||
|
Test that invalid template falls back to default sources template when
|
||||||
|
sources_dir exists.
|
||||||
|
"""
|
||||||
|
build_dir = tmp_path / "build"
|
||||||
|
build_dir.mkdir()
|
||||||
|
|
||||||
|
sources_dir = build_dir / "_sources"
|
||||||
|
sources_dir.mkdir()
|
||||||
|
|
||||||
|
# Mock app with html_sourcelink_suffix
|
||||||
|
class MockApp:
|
||||||
|
class Config:
|
||||||
|
html_sourcelink_suffix = ".txt"
|
||||||
|
|
||||||
|
config = Config()
|
||||||
|
|
||||||
|
# Invalid template with typo in variable name
|
||||||
|
config = {
|
||||||
|
"llms_txt_file": True,
|
||||||
|
"llms_txt_filename": "llms.txt",
|
||||||
|
"llms_txt_uri_template": "{base_urll}/{docname}",
|
||||||
|
"html_baseurl": "https://example.com",
|
||||||
|
}
|
||||||
|
writer = FileWriter(config, str(build_dir), MockApp())
|
||||||
|
|
||||||
|
page_titles = {
|
||||||
|
"index": "Home Page",
|
||||||
|
}
|
||||||
|
|
||||||
|
page_order = [("index", ".rst")]
|
||||||
|
|
||||||
|
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||||
|
|
||||||
|
verbose_file = build_dir / "llms.txt"
|
||||||
|
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||||
|
content = f.read()
|
||||||
|
|
||||||
|
# Should fallback to default sources template
|
||||||
|
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
|
||||||
Reference in New Issue
Block a user