Xarkam/sphinx-llms-txt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
Jared Dillard
2025-05-16 00:40:07 -07:00
parent 9aea054794
commit fc547ad526
8 changed files with 487 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.flake8
+12
View File
@@ -0,0 +1,12 @@
[flake8]
max-line-length = 88
extend-ignore = E203, W503
exclude =
.git,
__pycache__,
.venv,
venv,
.eggs,
*.egg,
build,
dist
.gitignore
+21
View File
@@ -0,0 +1,21 @@
*.pyc
.idea/
*.code-workspace
# Unit test / coverage reports
.tox
# Distribution / packaging
*.egg-info/
dist/
build/
# Environments
.venv
# Sphinx documentation
docs/_build/
# vale packages
docs/_vale/Microsoft/
docs/_vale/write-good/
.pre-commit-config.yaml
+27
View File
@@ -0,0 +1,27 @@
# Install pre-commit hooks via
# pre-commit install
repos:
- repo: https://github.com/PyCQA/isort
rev: 5.13.2
hooks:
- id: isort
- repo: https://github.com/psf/black
rev: 24.10.0
hooks:
- id: black
- repo: https://github.com/PyCQA/flake8
rev: 7.1.1
hooks:
- id: flake8
- repo: https://github.com/sphinx-contrib/sphinx-lint
rev: v1.0.0
hooks:
- id: sphinx-lint
args: [--jobs=1]
files: ^docs/|CHANGELOG.rst|README.rst
types: [rst]
CHANGELOG.rst
+7
View File
@@ -0,0 +1,7 @@
Changelog
=========
0.1.0
-----
- Initial release
MANIFEST.in
+1
View File
@@ -0,0 +1 @@
include LICENSE
README.md
+37
View File
@@ -0,0 +1,37 @@
# Sphinx llms-full.txt Extension
A Sphinx extension that creates a single combined documentation `llms-full.txt` file, written in reStructuredText.
## Installation
```bash
pip install sphinx-llms-txt
```
## Usage
1. Add the extension to your Sphinx configuration (`conf.py`):
```python
extensions = [
'sphinx_llms_txt',
]
```
## Configuration Options
### `llms_txt_filename`
- **Type**: string
- **Default**: `'llms-full.txt'`
- **Description**: Name of the output file
### `llms_txt_verbose`
- **Type**: boolean
- **Default**: `False`
- **Description**: Whether to include a summary in the build output
## License
MIT License - see LICENSE file for details.
pyproject.toml
+87
View File
@@ -0,0 +1,87 @@
[build-system]
requires = [
"setuptools",
]
build-backend = "setuptools.build_meta"
[project]
name = "sphinx-llms-txt"
description = "llms-full.txt generator for Sphinx"
authors = [
{name = "Jared Dillard", email = "jared.dillard@gmail.com"},
]
maintainers = [
{name = "Jared Dillard", email = "jared.dillard@gmail.com"},
]
classifiers = [
"Framework :: Sphinx :: Extension",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Documentation :: Sphinx",
]
license = {text = "MIT"}
readme = "README.md"
dynamic = ["version"]
[project.urls]
download = "https://pypi.org/project/sphinx-llms-txt/"
source = "https://github.com/jdillard/sphinx-llms-txt"
changelog = "https://github.com/jdillard/sphinx-llms-txt/blob/master/CHANGELOG.rst"
[project.optional-dependencies]
dev = [
"pytest>=7.0.0",
"black",
"flake8",
"mypy",
"isort",
"pre-commit",
]
test = [
"pytest>=7.0.0",
]
[tool.setuptools.dynamic]
version = {attr = "sphinx_llms_txt.__version__"}
[tool.black]
line-length = 88
target-version = ['py39', 'py310', 'py311', 'py312']
include = '\.pyi?$'
extend-exclude = '''
/(
# directories
\.eggs
| \.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
)/
'''
[tool.isort]
profile = "black"
[tool.mypy]
python_version = "3.9"
[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_functions = ["test_*"]
addopts = "-v"
filterwarnings = [
"error",
"ignore::UserWarning",
"ignore::DeprecationWarning",
]
sphinx_llms_txt/__init__.py
+295
View File
@@ -0,0 +1,295 @@
"""
Sphinx extension to create a combined sources file (llms-full.rst)
that combines all documentation sources in the correct build order.
"""
from pathlib import Path
from typing import Any, Dict, List
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.util import logging
__version__ = "0.1.0"
logger = logging.getLogger(__name__)
class LLMSFullManager:
"""Manages the collection and ordering of documentation sources."""
def __init__(self):
self.page_titles: Dict[str, str] = {}
self.config: Dict[str, Any] = {}
self.master_doc: str = None
self.env: BuildEnvironment = None
def set_master_doc(self, master_doc: str):
"""Set the master document name."""
self.master_doc = master_doc
def set_env(self, env: BuildEnvironment):
"""Set the Sphinx environment."""
self.env = env
def update_page_title(self, docname: str, title: str):
"""Update the title for a page."""
if title:
self.page_titles[docname] = title
def set_config(self, config: Dict[str, Any]):
"""Set configuration options."""
self.config = config
def get_page_order(self) -> List[str]:
"""Get the correct page order from the toctree structure."""
if not self.env or not self.master_doc:
return []
page_order = []
visited = set()
def collect_from_toctree(docname: str):
"""Recursively collect documents from toctree."""
if docname in visited:
return
visited.add(docname)
# Add the current document
if docname not in page_order:
page_order.append(docname)
# Check for toctree entries in this document
try:
# Look for toctree_includes which contains the direct children
if (
hasattr(self.env, "toctree_includes")
and docname in self.env.toctree_includes
):
for child_docname in self.env.toctree_includes[docname]:
collect_from_toctree(child_docname)
else:
# Fallback: try to resolve and parse the toctree
toctree = self.env.get_and_resolve_toctree(docname, None)
if toctree:
from docutils import nodes
for node in toctree.traverse(nodes.reference):
if "refuri" in node.attributes:
refuri = node.attributes["refuri"]
if refuri and refuri.endswith(".html"):
child_docname = refuri[:-5] # Remove .html
if (
child_docname != docname
): # Avoid circular references
collect_from_toctree(child_docname)
except Exception as e:
logger.debug(f"Could not get toctree for {docname}: {e}")
# Start from the master document
collect_from_toctree(self.master_doc)
# Add any remaining documents not in the toctree (sorted)
if hasattr(self.env, "all_docs"):
remaining = sorted(
[doc for doc in self.env.all_docs.keys() if doc not in page_order]
)
page_order.extend(remaining)
return page_order
def combine_sources(self, outdir: str, srcdir: str):
"""Combine all source files into a single file."""
# Get the correct page order
page_order = self.get_page_order()
if not page_order:
logger.warning(
"Could not determine page order, skipping llms-full creation"
)
return
# Determine output file name and location
output_filename = self.config.get("llms_txt_filename")
output_path = Path(outdir) / output_filename
# Find sources directory
sources_dir = None
possible_sources = [
Path(outdir) / "_sources",
Path(outdir) / "html" / "_sources",
Path(outdir) / "singlehtml" / "_sources",
]
for path in possible_sources:
if path.exists():
sources_dir = path
break
if not sources_dir:
logger.warning(
"Could not find _sources directory, skipping llms-full creation"
)
return
# Collect all available source files
txt_files = {}
for f in sources_dir.glob("*.txt"):
txt_files[f.stem] = f
# Create a mapping from docnames to actual file names
docname_to_file = {}
# Try exact matches first
for docname in page_order:
if docname in txt_files:
docname_to_file[docname] = txt_files[docname]
else:
# Try with .rst extension
if f"{docname}.rst" in txt_files:
docname_to_file[docname] = txt_files[f"{docname}.rst"]
# Try with .txt extension
elif f"{docname}.txt" in txt_files:
docname_to_file[docname] = txt_files[f"{docname}.txt"]
# Try with underscores instead of hyphens
elif docname.replace("-", "_") in txt_files:
docname_to_file[docname] = txt_files[docname.replace("-", "_")]
# Try with hyphens instead of underscores
elif docname.replace("_", "-") in txt_files:
docname_to_file[docname] = txt_files[docname.replace("_", "-")]
# Generate content
content_parts = []
# Add pages in order
added_files = set()
for docname in page_order:
if docname in docname_to_file:
file_path = docname_to_file[docname]
content = self._read_source_file(file_path, docname)
if content:
content_parts.append(content)
added_files.add(file_path.stem)
else:
logger.warning(f"Source file not found for: {docname}")
# Add any remaining files (in alphabetical order)
remaining_files = sorted(
[name for name in txt_files if name not in added_files]
)
if remaining_files:
logger.info(f"Adding remaining files: {remaining_files}")
for file_stem in remaining_files:
file_path = txt_files[file_stem]
content = self._read_source_file(file_path, file_stem)
if content:
content_parts.append(content)
# Write combined file
try:
with open(output_path, "w", encoding="utf-8") as f:
f.write("\n".join(content_parts))
logger.info(
f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources"
)
# Log summary information if requested
if self.config.get("llms_txt_verbose"):
self._log_summary_info(page_order)
except Exception as e:
logger.error(f"Error writing combined sources file: {e}")
def _read_source_file(self, file_path: Path, docname: str) -> str:
"""Read and format a single source file."""
try:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
section_lines = [content, ""]
return "\n".join(section_lines)
except Exception as e:
logger.error(f"Error reading source file {file_path}: {e}")
return ""
def _log_summary_info(self, page_order: List[str]):
"""Log summary information to the logger."""
logger.info("")
logger.info("llms-txt Summary")
logger.info("================")
logger.info(f"Total pages: {len(page_order)}")
logger.info(f"Configuration: {self.config}")
logger.info("Page order:")
for i, docname in enumerate(page_order, 1):
title = self.page_titles.get(docname, docname)
logger.info(f"{i:3d}. {docname} - {title}")
# Global manager instance
_manager = LLMSFullManager()
def doctree_resolved(app: Sphinx, doctree, docname: str):
"""Called when a docname has been resolved to a document."""
# Extract title from the document
from docutils import nodes
title = None
for node in doctree.traverse(nodes.title):
title = node.astext()
break
if title:
_manager.update_page_title(docname, title)
def build_finished(app: Sphinx, exception):
"""Called when the build is finished."""
if exception is None:
# Set the environment and master doc in the manager
_manager.set_env(app.env)
_manager.set_master_doc(app.config.master_doc)
# Set up configuration
config = {
"llms_txt_filename": app.config.llms_txt_filename,
"llms_txt_verbose": app.config.llms_txt_verbose,
}
_manager.set_config(config)
# Get final titles from the environment at build completion
if hasattr(app.env, "titles"):
for docname, title_node in app.env.titles.items():
if title_node:
title = title_node.astext()
_manager.update_page_title(docname, title)
# Create the combined file
_manager.combine_sources(app.outdir, app.srcdir)
def setup(app: Sphinx) -> Dict[str, Any]:
"""Set up the Sphinx extension."""
# Add configuration options
app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
app.add_config_value("llms_txt_verbose", False, "env")
# Connect to Sphinx events
app.connect("doctree-resolved", doctree_resolved)
app.connect("build-finished", build_finished)
# Reset manager for each build
global _manager
_manager = LLMSFullManager()
return {
"version": __version__,
"parallel_read_safe": True,
"parallel_write_safe": True,
}