Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5bfbb0168f | ||
|
|
e2a80faf04 | ||
|
|
b63801bcff | ||
|
|
c45ebb0369 | ||
|
|
f5dcd15889 |
@@ -10,9 +10,9 @@ jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- uses: pre-commit/action@v3.0.1
|
||||
@@ -23,17 +23,17 @@ jobs:
|
||||
python-version: ['3.9', '3.10', '3.11', '3.12']
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v5
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e ".[dev]"
|
||||
pip install -e . --group dev
|
||||
|
||||
# - name: Run mypy
|
||||
# run: |
|
||||
|
||||
+10
-11
@@ -1,15 +1,14 @@
|
||||
version: 2
|
||||
|
||||
build:
|
||||
os: "ubuntu-20.04"
|
||||
os: ubuntu-24.04
|
||||
tools:
|
||||
python: "3.10"
|
||||
|
||||
sphinx:
|
||||
configuration: docs/source/conf.py
|
||||
|
||||
python:
|
||||
install:
|
||||
- requirements: docs/requirements.txt
|
||||
- method: pip
|
||||
path: .
|
||||
python: "3.13"
|
||||
commands:
|
||||
- pip install cmake
|
||||
- pip install -r docs/requirements.txt
|
||||
- pip install -e .
|
||||
- cmake --workflow --preset documentation-workflow
|
||||
# Copy built documentation to Read the Docs output directory
|
||||
- mkdir -p $READTHEDOCS_OUTPUT/html
|
||||
- cp -r build/* $READTHEDOCS_OUTPUT/html/
|
||||
|
||||
@@ -1,6 +1,24 @@
|
||||
Changelog
|
||||
=========
|
||||
|
||||
0.7.0
|
||||
-----
|
||||
|
||||
- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`.
|
||||
`#48 <https://github.com/jdillard/sphinx-llms-txt/pull/48>`_
|
||||
|
||||
0.6.0
|
||||
-----
|
||||
|
||||
- Improve _sources directory handling
|
||||
`#47 <https://github.com/jdillard/sphinx-llms-txt/pull/47>`_
|
||||
|
||||
0.5.3
|
||||
-----
|
||||
|
||||
- Make sphinx a required dependency since there are imports from Sphinx
|
||||
`#44 <https://github.com/jdillard/sphinx-llms-txt/pull/44>`_
|
||||
|
||||
0.5.2
|
||||
-----
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
|
||||
|
||||
# Add CMake module path
|
||||
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||
|
||||
# Add documentation
|
||||
add_subdirectory(docs)
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"version": 6,
|
||||
"configurePresets": [
|
||||
{
|
||||
"name": "documentation",
|
||||
"displayName": "Documentation Build",
|
||||
"description": "Configure project with documentation environment setup",
|
||||
"binaryDir": "${sourceDir}/build"
|
||||
}
|
||||
],
|
||||
"buildPresets": [
|
||||
{
|
||||
"name": "html",
|
||||
"displayName": "Build HTML Documentation",
|
||||
"configurePreset": "documentation",
|
||||
"targets": ["html"]
|
||||
},
|
||||
{
|
||||
"name": "markdown",
|
||||
"displayName": "Build Markdown Documentation",
|
||||
"configurePreset": "documentation",
|
||||
"targets": ["markdown"]
|
||||
},
|
||||
{
|
||||
"name": "docs-parallel",
|
||||
"displayName": "Build HTML and Markdown in parallel",
|
||||
"configurePreset": "documentation",
|
||||
"targets": ["html", "markdown"]
|
||||
}
|
||||
],
|
||||
"workflowPresets": [
|
||||
{
|
||||
"name": "documentation-workflow",
|
||||
"displayName": "Documentation Build Workflow",
|
||||
"steps": [
|
||||
{
|
||||
"type": "configure",
|
||||
"name": "documentation"
|
||||
},
|
||||
{
|
||||
"type": "build",
|
||||
"name": "docs-parallel"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# SetupSphinx.cmake - Find Sphinx for CI environments
|
||||
|
||||
# Function to find Sphinx in the system
|
||||
function(setup_sphinx_environment)
|
||||
# Find sphinx-build executable in system
|
||||
find_program(SPHINX_EXECUTABLE
|
||||
NAMES sphinx-build
|
||||
DOC "Sphinx documentation generator"
|
||||
)
|
||||
|
||||
if(NOT SPHINX_EXECUTABLE)
|
||||
message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
|
||||
endif()
|
||||
|
||||
# Get Sphinx version
|
||||
execute_process(
|
||||
COMMAND ${SPHINX_EXECUTABLE} --version
|
||||
OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
|
||||
ERROR_VARIABLE SPHINX_VERSION_OUTPUT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
|
||||
message(STATUS "${SPHINX_VERSION_OUTPUT}")
|
||||
|
||||
# Export to parent scope
|
||||
set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
@@ -0,0 +1,24 @@
|
||||
# Find Sphinx
|
||||
include(SetupSphinx)
|
||||
setup_sphinx_environment()
|
||||
|
||||
set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
|
||||
set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
|
||||
|
||||
# Build HTML documentation
|
||||
add_custom_target(html
|
||||
COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
||||
COMMENT "Building HTML documentation"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
# Build markdown documentation
|
||||
add_custom_target(markdown
|
||||
COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
|
||||
COMMENT "Building Markdown documentation"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
# Build both formats
|
||||
add_custom_target(docs ALL)
|
||||
add_dependencies(docs html markdown)
|
||||
@@ -4,3 +4,4 @@ sphinx-contributors
|
||||
sphinx
|
||||
sphinx-llms-txt
|
||||
sphinxext-opengraph
|
||||
sphinx-markdown-builder
|
||||
|
||||
@@ -261,6 +261,37 @@ If you want to include absolute URLs for resources in your documentation, you ca
|
||||
|
||||
When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
|
||||
|
||||
.. _customizing_uri_links:
|
||||
|
||||
Customizing URI Links in llms.txt
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available.
|
||||
You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Default: Link to source files, if _sources exists
|
||||
llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||
|
||||
# Default: Link to HTML pages instead, if _sources doesn't exist
|
||||
llms_txt_uri_template = "{base_url}{docname}.html"
|
||||
|
||||
# Manual: Link to a custom markdown build
|
||||
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||
|
||||
.. _available_template_variables:
|
||||
|
||||
Available Template Variables
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Your URI template can use the following variables:
|
||||
|
||||
- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash)
|
||||
- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``)
|
||||
- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
|
||||
- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
|
||||
|
||||
.. _integration_examples:
|
||||
|
||||
Integration Examples
|
||||
@@ -285,6 +316,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
|
||||
This is a comprehensive documentation set for our project.
|
||||
It includes API references, usage examples, and tutorials.
|
||||
"""
|
||||
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||
|
||||
# Path handling
|
||||
html_baseurl = "https://docs.example.com/"
|
||||
|
||||
+2
-1
@@ -15,6 +15,7 @@ import subprocess
|
||||
project = "sphinx-llms-txt"
|
||||
copyright = "Jared Dillard"
|
||||
author = "Jared Dillard"
|
||||
llms_txt_uri_template = "{base_url}{docname}.md"
|
||||
llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
|
||||
llms_txt_summary = """
|
||||
A Sphinx extension that generates a summary llms.txt file,written in Markdown,
|
||||
@@ -88,7 +89,7 @@ html_theme_options = {
|
||||
"source_directory": "docs/source/",
|
||||
}
|
||||
|
||||
html_baseurl = "https://sphinx-llms-txt.readthedocs.org/"
|
||||
html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/"
|
||||
|
||||
|
||||
# -- Options for HTMLHelp output ---------------------------------------------
|
||||
|
||||
@@ -58,6 +58,15 @@ Project Configuration Values
|
||||
|
||||
.. versionadded:: 0.2.0
|
||||
|
||||
.. confval:: llms_txt_uri_template
|
||||
|
||||
- **Type**: string or ``None``
|
||||
- **Default**: ``None``
|
||||
- **Description**: Template string for generating URIs in ``llms.txt``.
|
||||
See :ref:`customizing_uri_links`.
|
||||
|
||||
.. versionadded:: 0.7.0
|
||||
|
||||
.. confval:: llms_txt_directives
|
||||
|
||||
- **Type**: list of strings
|
||||
|
||||
@@ -19,7 +19,7 @@ Local development
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
pip install -e ".[dev]"
|
||||
pip install -e . --group dev
|
||||
|
||||
#. Install pre-commit Git hook scripts:
|
||||
|
||||
|
||||
+6
-2
@@ -26,13 +26,16 @@ classifiers = [
|
||||
license = {text = "MIT"}
|
||||
readme = "README.md"
|
||||
dynamic = ["version"]
|
||||
dependencies = [
|
||||
"sphinx",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
download = "https://pypi.org/project/sphinx-llms-txt/"
|
||||
source = "https://github.com/jdillard/sphinx-llms-txt"
|
||||
changelog = "https://github.com/jdillard/sphinx-llms-txt/blob/master/CHANGELOG.rst"
|
||||
|
||||
[project.optional-dependencies]
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=7.0.0",
|
||||
"black",
|
||||
@@ -40,12 +43,13 @@ dev = [
|
||||
"mypy",
|
||||
"isort",
|
||||
"pre-commit",
|
||||
"sphinx",
|
||||
]
|
||||
test = [
|
||||
"pytest>=7.0.0",
|
||||
]
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["sphinx_llms_txt"]
|
||||
|
||||
[tool.setuptools.dynamic]
|
||||
version = {attr = "sphinx_llms_txt.__version__"}
|
||||
|
||||
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
|
||||
from .processor import DocumentProcessor
|
||||
from .writer import FileWriter
|
||||
|
||||
__version__ = "0.5.2"
|
||||
__version__ = "0.7.0"
|
||||
|
||||
# Export classes needed by tests
|
||||
__all__ = [
|
||||
@@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception):
|
||||
config = {
|
||||
"llms_txt_file": app.config.llms_txt_file,
|
||||
"llms_txt_filename": app.config.llms_txt_filename,
|
||||
"llms_txt_uri_template": app.config.llms_txt_uri_template,
|
||||
"llms_txt_title": app.config.llms_txt_title,
|
||||
"llms_txt_summary": summary,
|
||||
"llms_txt_full_file": app.config.llms_txt_full_file,
|
||||
@@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
|
||||
|
||||
app.add_config_value("llms_txt_file", True, "env")
|
||||
app.add_config_value("llms_txt_filename", "llms.txt", "env")
|
||||
app.add_config_value("llms_txt_uri_template", None, "env")
|
||||
app.add_config_value("llms_txt_full_file", True, "env")
|
||||
app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
|
||||
app.add_config_value("llms_txt_full_max_size", None, "env")
|
||||
|
||||
+35
-12
@@ -204,25 +204,43 @@ class LLMSFullManager:
|
||||
sources_dir = path
|
||||
break
|
||||
|
||||
if not sources_dir:
|
||||
logger.warning(
|
||||
"Could not find _sources directory, skipping llms-full creation"
|
||||
)
|
||||
return
|
||||
|
||||
# Get the correct page order with source suffixes
|
||||
# Get the correct page order (with or without source suffixes)
|
||||
page_order = self.collector.get_page_order(sources_dir)
|
||||
|
||||
if not page_order:
|
||||
logger.warning(
|
||||
"Could not determine page order, skipping llms-full creation"
|
||||
)
|
||||
logger.warning("Could not determine page order, skipping file generation")
|
||||
return
|
||||
|
||||
# Apply exclusion filter if configured
|
||||
page_order = self.collector.filter_excluded_pages(page_order)
|
||||
|
||||
# Determine output file name and location
|
||||
# If no sources directory, only generate llms.txt and return early
|
||||
if not sources_dir:
|
||||
# Generate llms.txt if requested
|
||||
if self.config.get("llms_txt_file"):
|
||||
filtered_page_order = self._filter_ignored_pages(page_order)
|
||||
self.writer.write_verbose_info_to_file(
|
||||
filtered_page_order,
|
||||
self.collector.page_titles,
|
||||
0, # No line count since no llms-full.txt
|
||||
sources_dir,
|
||||
)
|
||||
|
||||
# Only warn if user explicitly wants llms-full.txt
|
||||
if self.config.get("llms_txt_full_file"):
|
||||
# Check if html_copy_source is False
|
||||
if self.app and not self.app.config.html_copy_source:
|
||||
logger.warning(
|
||||
"Could not find _sources directory, skipping llms-full.txt."
|
||||
"Set html_copy_source = True in conf.py to enable."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Could not find _sources directory, skipping llms-full.txt"
|
||||
)
|
||||
return
|
||||
|
||||
# Determine output file name and location for llms-full.txt
|
||||
output_filename = self.config.get("llms_txt_full_filename")
|
||||
output_path = Path(outdir) / output_filename
|
||||
|
||||
@@ -506,6 +524,7 @@ class LLMSFullManager:
|
||||
filtered_page_order,
|
||||
self.collector.page_titles,
|
||||
total_line_count,
|
||||
sources_dir,
|
||||
)
|
||||
return
|
||||
elif action == "note":
|
||||
@@ -519,6 +538,7 @@ class LLMSFullManager:
|
||||
filtered_page_order,
|
||||
self.collector.page_titles,
|
||||
total_line_count,
|
||||
sources_dir,
|
||||
)
|
||||
return
|
||||
elif action == "keep":
|
||||
@@ -537,7 +557,10 @@ class LLMSFullManager:
|
||||
if success and self.config.get("llms_txt_file"):
|
||||
filtered_page_order = self._filter_ignored_pages(page_order)
|
||||
self.writer.write_verbose_info_to_file(
|
||||
filtered_page_order, self.collector.page_titles, total_line_count
|
||||
filtered_page_order,
|
||||
self.collector.page_titles,
|
||||
total_line_count,
|
||||
sources_dir,
|
||||
)
|
||||
|
||||
def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:
|
||||
|
||||
@@ -19,6 +19,42 @@ class FileWriter:
|
||||
self.outdir = outdir
|
||||
self.app = app
|
||||
|
||||
def _resolve_uri_template(self, sources_dir: Path = None) -> str:
|
||||
"""Resolve which URI template to use based on configuration and sources_dir.
|
||||
|
||||
Args:
|
||||
sources_dir: Path to _sources directory (None if not found)
|
||||
|
||||
Returns:
|
||||
The template string to use for generating URIs
|
||||
"""
|
||||
# If custom template exists
|
||||
custom_template = self.config.get("llms_txt_uri_template")
|
||||
|
||||
if custom_template:
|
||||
# Validate user's template by checking for valid variable names
|
||||
try:
|
||||
# Try formatting with test valid values to validate syntax
|
||||
test_values = {
|
||||
"base_url": "http://example.com/",
|
||||
"docname": "test",
|
||||
"suffix": ".rst",
|
||||
"sourcelink_suffix": ".txt",
|
||||
}
|
||||
custom_template.format(**test_values)
|
||||
return custom_template
|
||||
except (KeyError, ValueError) as e:
|
||||
logger.warning(
|
||||
f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. "
|
||||
f"Falling back to default."
|
||||
)
|
||||
|
||||
# Else, use one of the default templates
|
||||
if sources_dir:
|
||||
return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||
else:
|
||||
return "{base_url}{docname}.html"
|
||||
|
||||
def write_combined_file(
|
||||
self, content_parts: List[str], output_path: Path, total_line_count: int
|
||||
) -> bool:
|
||||
@@ -50,6 +86,7 @@ class FileWriter:
|
||||
page_order: Union[List[str], List[Tuple[str, str]]],
|
||||
page_titles: Dict[str, str],
|
||||
total_line_count: int = 0,
|
||||
sources_dir: Path = None,
|
||||
) -> bool:
|
||||
"""Write summary information to the llms.txt file.
|
||||
|
||||
@@ -57,6 +94,7 @@ class FileWriter:
|
||||
page_order: Ordered list of document names or (docname, suffix) tuples
|
||||
page_titles: Dictionary mapping docnames to titles
|
||||
total_line_count: Total number of lines in the combined content
|
||||
sources_dir: Path to _sources directory (None if not found)
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
@@ -102,14 +140,37 @@ class FileWriter:
|
||||
if not base_url.endswith("/"):
|
||||
base_url += "/"
|
||||
|
||||
# Get sourcelink suffix from Sphinx config
|
||||
sourcelink_suffix = ""
|
||||
if self.app and hasattr(self.app.config, "html_sourcelink_suffix"):
|
||||
sourcelink_suffix = self.app.config.html_sourcelink_suffix
|
||||
# Handle empty string case specially
|
||||
if sourcelink_suffix == "":
|
||||
sourcelink_suffix = "" # Keep it empty
|
||||
elif not sourcelink_suffix.startswith("."):
|
||||
sourcelink_suffix = "." + sourcelink_suffix
|
||||
|
||||
# Resolve which template to use
|
||||
uri_template = self._resolve_uri_template(sources_dir)
|
||||
|
||||
for item in page_order:
|
||||
# Handle both old format (str) and new format (tuple)
|
||||
if isinstance(item, tuple):
|
||||
docname, _ = item
|
||||
docname, suffix = item
|
||||
else:
|
||||
docname = item
|
||||
suffix = None
|
||||
|
||||
title = page_titles.get(docname, docname)
|
||||
f.write(f"- [{title}]({base_url}{docname}.html)\n")
|
||||
|
||||
uri = uri_template.format(
|
||||
base_url=base_url,
|
||||
docname=docname,
|
||||
suffix=suffix or "",
|
||||
sourcelink_suffix=sourcelink_suffix,
|
||||
)
|
||||
|
||||
f.write(f"- [{title}]({uri})\n")
|
||||
|
||||
logger.info(f"sphinx-llms-txt: created {output_path}")
|
||||
return True
|
||||
|
||||
@@ -794,6 +794,7 @@ def test_summary_default_uses_first_paragraph():
|
||||
llms_txt_summary = None # Not configured
|
||||
llms_txt_file = True
|
||||
llms_txt_filename = "llms.txt"
|
||||
llms_txt_uri_template = None
|
||||
llms_txt_title = None
|
||||
llms_txt_full_file = True
|
||||
llms_txt_full_filename = "llms-full.txt"
|
||||
@@ -1031,3 +1032,128 @@ def test_code_files_ignored_patterns(tmp_path, caplog):
|
||||
assert (
|
||||
"Code file pattern 'docs/**/*.rst' ignored." in captured_warnings[0]
|
||||
), f"Warning message should contain expected text. Got: {captured_warnings[0]}"
|
||||
|
||||
|
||||
def test_llms_txt_generated_without_sources_dir(tmp_path):
|
||||
"""Test that llms.txt is generated even when _sources directory doesn't exist."""
|
||||
|
||||
from sphinx_llms_txt.manager import LLMSFullManager
|
||||
|
||||
# Create manager
|
||||
manager = LLMSFullManager()
|
||||
|
||||
# Set config to enable llms.txt
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
"llms_txt_full_file": True,
|
||||
"llms_txt_full_filename": "llms-full.txt",
|
||||
"llms_txt_exclude": [],
|
||||
"llms_txt_directives": [],
|
||||
}
|
||||
manager.set_config(config)
|
||||
|
||||
# Create directories (but no _sources)
|
||||
outdir = tmp_path / "build"
|
||||
srcdir = tmp_path / "source"
|
||||
outdir.mkdir()
|
||||
srcdir.mkdir()
|
||||
|
||||
# Mock env with documents
|
||||
class MockEnv:
|
||||
all_docs = {"index": None, "about": None}
|
||||
titles = {
|
||||
"index": type("TitleNode", (), {"astext": lambda self: "Home"})(),
|
||||
"about": type("TitleNode", (), {"astext": lambda self: "About"})(),
|
||||
}
|
||||
toctree_includes = {"index": ["about"]}
|
||||
|
||||
manager.set_env(MockEnv())
|
||||
manager.set_master_doc("index")
|
||||
|
||||
# Update page titles directly in the collector
|
||||
manager.update_page_title("index", "Home")
|
||||
manager.update_page_title("about", "About")
|
||||
|
||||
# Call combine_sources - should generate llms.txt even without _sources
|
||||
manager.combine_sources(str(outdir), str(srcdir))
|
||||
|
||||
# Verify llms.txt was created
|
||||
llms_txt = outdir / "llms.txt"
|
||||
assert llms_txt.exists(), "llms.txt should be generated even without _sources"
|
||||
|
||||
# Verify llms-full.txt was NOT created (since no _sources)
|
||||
llms_full_txt = outdir / "llms-full.txt"
|
||||
assert (
|
||||
not llms_full_txt.exists()
|
||||
), "llms-full.txt should not be generated without _sources"
|
||||
|
||||
# Read llms.txt and verify it has content
|
||||
with open(llms_txt, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# Should contain page titles and links
|
||||
assert "Home" in content
|
||||
assert "About" in content
|
||||
assert "index.html" in content
|
||||
assert "about.html" in content
|
||||
|
||||
|
||||
def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
|
||||
"""
|
||||
Test that no warning is logged when llms_txt_full_file=False and
|
||||
_sources doesn't exist.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from sphinx_llms_txt.manager import LLMSFullManager
|
||||
|
||||
# Create manager
|
||||
manager = LLMSFullManager()
|
||||
|
||||
# Set config with llms_txt_full_file=False
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
"llms_txt_full_file": False, # User doesn't want llms-full.txt
|
||||
"llms_txt_full_filename": "llms-full.txt",
|
||||
"llms_txt_exclude": [],
|
||||
"llms_txt_directives": [],
|
||||
}
|
||||
manager.set_config(config)
|
||||
|
||||
# Create directories (but no _sources)
|
||||
outdir = tmp_path / "build"
|
||||
srcdir = tmp_path / "source"
|
||||
outdir.mkdir()
|
||||
srcdir.mkdir()
|
||||
|
||||
# Mock env with documents
|
||||
class MockEnv:
|
||||
all_docs = {"index": None}
|
||||
titles = {"index": type("TitleNode", (), {"astext": lambda self: "Home"})()}
|
||||
toctree_includes = {"index": []}
|
||||
|
||||
manager.set_env(MockEnv())
|
||||
manager.set_master_doc("index")
|
||||
manager.update_page_title("index", "Home")
|
||||
|
||||
# Capture warnings
|
||||
captured_warnings = []
|
||||
|
||||
def capture_warning(message, *args, **kwargs):
|
||||
if "_sources" in str(message):
|
||||
captured_warnings.append(message)
|
||||
|
||||
with patch("sphinx_llms_txt.manager.logger.warning", side_effect=capture_warning):
|
||||
# Call combine_sources
|
||||
manager.combine_sources(str(outdir), str(srcdir))
|
||||
|
||||
# Verify NO warning was logged since llms_txt_full_file=False
|
||||
assert (
|
||||
len(captured_warnings) == 0
|
||||
), "No warning should be logged when llms_txt_full_file=False"
|
||||
|
||||
# Verify llms.txt was still created
|
||||
llms_txt = outdir / "llms.txt"
|
||||
assert llms_txt.exists()
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Test URI template functionality for llms.txt links."""
|
||||
|
||||
from sphinx_llms_txt import FileWriter
|
||||
|
||||
|
||||
def test_uri_template_with_sources_dir(tmp_path):
|
||||
"""Test that default template uses _sources links when sources_dir exists."""
|
||||
build_dir = tmp_path / "build"
|
||||
build_dir.mkdir()
|
||||
|
||||
# Create _sources directory to simulate its existence
|
||||
sources_dir = build_dir / "_sources"
|
||||
sources_dir.mkdir()
|
||||
|
||||
# Mock app with html_sourcelink_suffix
|
||||
class MockApp:
|
||||
class Config:
|
||||
html_sourcelink_suffix = ".txt"
|
||||
|
||||
config = Config()
|
||||
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
"llms_txt_uri_template": (
|
||||
"{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
|
||||
),
|
||||
"html_baseurl": "https://example.com",
|
||||
}
|
||||
writer = FileWriter(config, str(build_dir), MockApp())
|
||||
|
||||
page_titles = {
|
||||
"index": "Home Page",
|
||||
"about": "About Us",
|
||||
}
|
||||
|
||||
# Page order with suffixes (simulating _sources files exist)
|
||||
page_order = [("index", ".rst"), ("about", ".md")]
|
||||
|
||||
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||
|
||||
# Check that the file was created
|
||||
verbose_file = build_dir / "llms.txt"
|
||||
assert verbose_file.exists()
|
||||
|
||||
# Read the file content
|
||||
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# Should link to _sources files
|
||||
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
|
||||
assert "- [About Us](https://example.com/_sources/about.md.txt)" in content
|
||||
|
||||
|
||||
def test_uri_template_without_sources_dir(tmp_path):
|
||||
"""
|
||||
Test that HTML template is used when sources_dir doesn't exist and no custom
|
||||
template.
|
||||
"""
|
||||
build_dir = tmp_path / "build"
|
||||
build_dir.mkdir()
|
||||
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
# No custom template set
|
||||
"html_baseurl": "https://example.com",
|
||||
}
|
||||
writer = FileWriter(config, str(build_dir))
|
||||
|
||||
page_titles = {
|
||||
"index": "Home Page",
|
||||
"about": "About Us",
|
||||
}
|
||||
|
||||
# Page order without suffixes (simulating no _sources)
|
||||
page_order = [("index", None), ("about", None)]
|
||||
|
||||
# Pass None for sources_dir to simulate it doesn't exist
|
||||
writer.write_verbose_info_to_file(page_order, page_titles, 0, None)
|
||||
|
||||
# Check that the file was created
|
||||
verbose_file = build_dir / "llms.txt"
|
||||
assert verbose_file.exists()
|
||||
|
||||
# Read the file content
|
||||
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# Should fallback to HTML links
|
||||
assert "- [Home Page](https://example.com/index.html)" in content
|
||||
assert "- [About Us](https://example.com/about.html)" in content
|
||||
|
||||
|
||||
def test_uri_template_custom(tmp_path):
|
||||
"""Test that custom URI template works correctly."""
|
||||
build_dir = tmp_path / "build"
|
||||
build_dir.mkdir()
|
||||
|
||||
sources_dir = build_dir / "_sources"
|
||||
sources_dir.mkdir()
|
||||
|
||||
# Mock app with html_sourcelink_suffix
|
||||
class MockApp:
|
||||
class Config:
|
||||
html_sourcelink_suffix = ".txt"
|
||||
|
||||
config = Config()
|
||||
|
||||
# Custom template that uses different path
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
"llms_txt_uri_template": "{base_url}raw/{docname}{suffix}",
|
||||
"html_baseurl": "https://example.com/",
|
||||
}
|
||||
writer = FileWriter(config, str(build_dir), MockApp())
|
||||
|
||||
page_titles = {
|
||||
"index": "Home Page",
|
||||
}
|
||||
|
||||
page_order = [("index", ".rst")]
|
||||
|
||||
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||
|
||||
verbose_file = build_dir / "llms.txt"
|
||||
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# Should use custom template
|
||||
assert "- [Home Page](https://example.com/raw/index.rst)" in content
|
||||
|
||||
|
||||
def test_uri_template_invalid_fallback(tmp_path):
|
||||
"""
|
||||
Test that invalid template falls back to default sources template when
|
||||
sources_dir exists.
|
||||
"""
|
||||
build_dir = tmp_path / "build"
|
||||
build_dir.mkdir()
|
||||
|
||||
sources_dir = build_dir / "_sources"
|
||||
sources_dir.mkdir()
|
||||
|
||||
# Mock app with html_sourcelink_suffix
|
||||
class MockApp:
|
||||
class Config:
|
||||
html_sourcelink_suffix = ".txt"
|
||||
|
||||
config = Config()
|
||||
|
||||
# Invalid template with typo in variable name
|
||||
config = {
|
||||
"llms_txt_file": True,
|
||||
"llms_txt_filename": "llms.txt",
|
||||
"llms_txt_uri_template": "{base_urll}/{docname}",
|
||||
"html_baseurl": "https://example.com",
|
||||
}
|
||||
writer = FileWriter(config, str(build_dir), MockApp())
|
||||
|
||||
page_titles = {
|
||||
"index": "Home Page",
|
||||
}
|
||||
|
||||
page_order = [("index", ".rst")]
|
||||
|
||||
writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
|
||||
|
||||
verbose_file = build_dir / "llms.txt"
|
||||
with open(verbose_file, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
# Should fallback to default sources template
|
||||
assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
|
||||
Reference in New Issue
Block a user