Initial commit
This commit is contained in:
Jared Dillard
@@ -0,0 +1,295 @@
|
||||
"""
|
||||
Sphinx extension to create a combined sources file (llms-full.rst)
|
||||
that combines all documentation sources in the correct build order.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from sphinx.application import Sphinx
|
||||
from sphinx.environment import BuildEnvironment
|
||||
from sphinx.util import logging
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMSFullManager:
|
||||
"""Manages the collection and ordering of documentation sources."""
|
||||
|
||||
def __init__(self):
|
||||
self.page_titles: Dict[str, str] = {}
|
||||
self.config: Dict[str, Any] = {}
|
||||
self.master_doc: str = None
|
||||
self.env: BuildEnvironment = None
|
||||
|
||||
def set_master_doc(self, master_doc: str):
|
||||
"""Set the master document name."""
|
||||
self.master_doc = master_doc
|
||||
|
||||
def set_env(self, env: BuildEnvironment):
|
||||
"""Set the Sphinx environment."""
|
||||
self.env = env
|
||||
|
||||
def update_page_title(self, docname: str, title: str):
|
||||
"""Update the title for a page."""
|
||||
if title:
|
||||
self.page_titles[docname] = title
|
||||
|
||||
def set_config(self, config: Dict[str, Any]):
|
||||
"""Set configuration options."""
|
||||
self.config = config
|
||||
|
||||
def get_page_order(self) -> List[str]:
|
||||
"""Get the correct page order from the toctree structure."""
|
||||
if not self.env or not self.master_doc:
|
||||
return []
|
||||
|
||||
page_order = []
|
||||
visited = set()
|
||||
|
||||
def collect_from_toctree(docname: str):
|
||||
"""Recursively collect documents from toctree."""
|
||||
if docname in visited:
|
||||
return
|
||||
|
||||
visited.add(docname)
|
||||
|
||||
# Add the current document
|
||||
if docname not in page_order:
|
||||
page_order.append(docname)
|
||||
|
||||
# Check for toctree entries in this document
|
||||
try:
|
||||
# Look for toctree_includes which contains the direct children
|
||||
if (
|
||||
hasattr(self.env, "toctree_includes")
|
||||
and docname in self.env.toctree_includes
|
||||
):
|
||||
for child_docname in self.env.toctree_includes[docname]:
|
||||
collect_from_toctree(child_docname)
|
||||
else:
|
||||
# Fallback: try to resolve and parse the toctree
|
||||
toctree = self.env.get_and_resolve_toctree(docname, None)
|
||||
if toctree:
|
||||
from docutils import nodes
|
||||
|
||||
for node in toctree.traverse(nodes.reference):
|
||||
if "refuri" in node.attributes:
|
||||
refuri = node.attributes["refuri"]
|
||||
if refuri and refuri.endswith(".html"):
|
||||
child_docname = refuri[:-5] # Remove .html
|
||||
if (
|
||||
child_docname != docname
|
||||
): # Avoid circular references
|
||||
collect_from_toctree(child_docname)
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not get toctree for {docname}: {e}")
|
||||
|
||||
# Start from the master document
|
||||
collect_from_toctree(self.master_doc)
|
||||
|
||||
# Add any remaining documents not in the toctree (sorted)
|
||||
if hasattr(self.env, "all_docs"):
|
||||
remaining = sorted(
|
||||
[doc for doc in self.env.all_docs.keys() if doc not in page_order]
|
||||
)
|
||||
page_order.extend(remaining)
|
||||
|
||||
return page_order
|
||||
|
||||
def combine_sources(self, outdir: str, srcdir: str):
|
||||
"""Combine all source files into a single file."""
|
||||
# Get the correct page order
|
||||
page_order = self.get_page_order()
|
||||
|
||||
if not page_order:
|
||||
logger.warning(
|
||||
"Could not determine page order, skipping llms-full creation"
|
||||
)
|
||||
return
|
||||
|
||||
# Determine output file name and location
|
||||
output_filename = self.config.get("llms_txt_filename")
|
||||
output_path = Path(outdir) / output_filename
|
||||
|
||||
# Find sources directory
|
||||
sources_dir = None
|
||||
possible_sources = [
|
||||
Path(outdir) / "_sources",
|
||||
Path(outdir) / "html" / "_sources",
|
||||
Path(outdir) / "singlehtml" / "_sources",
|
||||
]
|
||||
|
||||
for path in possible_sources:
|
||||
if path.exists():
|
||||
sources_dir = path
|
||||
break
|
||||
|
||||
if not sources_dir:
|
||||
logger.warning(
|
||||
"Could not find _sources directory, skipping llms-full creation"
|
||||
)
|
||||
return
|
||||
|
||||
# Collect all available source files
|
||||
txt_files = {}
|
||||
for f in sources_dir.glob("*.txt"):
|
||||
txt_files[f.stem] = f
|
||||
|
||||
# Create a mapping from docnames to actual file names
|
||||
docname_to_file = {}
|
||||
|
||||
# Try exact matches first
|
||||
for docname in page_order:
|
||||
if docname in txt_files:
|
||||
docname_to_file[docname] = txt_files[docname]
|
||||
else:
|
||||
# Try with .rst extension
|
||||
if f"{docname}.rst" in txt_files:
|
||||
docname_to_file[docname] = txt_files[f"{docname}.rst"]
|
||||
# Try with .txt extension
|
||||
elif f"{docname}.txt" in txt_files:
|
||||
docname_to_file[docname] = txt_files[f"{docname}.txt"]
|
||||
# Try with underscores instead of hyphens
|
||||
elif docname.replace("-", "_") in txt_files:
|
||||
docname_to_file[docname] = txt_files[docname.replace("-", "_")]
|
||||
# Try with hyphens instead of underscores
|
||||
elif docname.replace("_", "-") in txt_files:
|
||||
docname_to_file[docname] = txt_files[docname.replace("_", "-")]
|
||||
|
||||
# Generate content
|
||||
content_parts = []
|
||||
|
||||
# Add pages in order
|
||||
added_files = set()
|
||||
|
||||
for docname in page_order:
|
||||
if docname in docname_to_file:
|
||||
file_path = docname_to_file[docname]
|
||||
content = self._read_source_file(file_path, docname)
|
||||
if content:
|
||||
content_parts.append(content)
|
||||
added_files.add(file_path.stem)
|
||||
else:
|
||||
logger.warning(f"Source file not found for: {docname}")
|
||||
|
||||
# Add any remaining files (in alphabetical order)
|
||||
remaining_files = sorted(
|
||||
[name for name in txt_files if name not in added_files]
|
||||
)
|
||||
if remaining_files:
|
||||
logger.info(f"Adding remaining files: {remaining_files}")
|
||||
for file_stem in remaining_files:
|
||||
file_path = txt_files[file_stem]
|
||||
content = self._read_source_file(file_path, file_stem)
|
||||
if content:
|
||||
content_parts.append(content)
|
||||
|
||||
# Write combined file
|
||||
try:
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(content_parts))
|
||||
|
||||
logger.info(
|
||||
f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources"
|
||||
)
|
||||
|
||||
# Log summary information if requested
|
||||
if self.config.get("llms_txt_verbose"):
|
||||
self._log_summary_info(page_order)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing combined sources file: {e}")
|
||||
|
||||
def _read_source_file(self, file_path: Path, docname: str) -> str:
|
||||
"""Read and format a single source file."""
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
content = f.read()
|
||||
|
||||
section_lines = [content, ""]
|
||||
|
||||
return "\n".join(section_lines)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading source file {file_path}: {e}")
|
||||
return ""
|
||||
|
||||
def _log_summary_info(self, page_order: List[str]):
|
||||
"""Log summary information to the logger."""
|
||||
logger.info("")
|
||||
logger.info("llms-txt Summary")
|
||||
logger.info("================")
|
||||
logger.info(f"Total pages: {len(page_order)}")
|
||||
logger.info(f"Configuration: {self.config}")
|
||||
logger.info("Page order:")
|
||||
for i, docname in enumerate(page_order, 1):
|
||||
title = self.page_titles.get(docname, docname)
|
||||
logger.info(f"{i:3d}. {docname} - {title}")
|
||||
|
||||
|
||||
# Global manager instance
|
||||
_manager = LLMSFullManager()
|
||||
|
||||
|
||||
def doctree_resolved(app: Sphinx, doctree, docname: str):
|
||||
"""Called when a docname has been resolved to a document."""
|
||||
# Extract title from the document
|
||||
from docutils import nodes
|
||||
|
||||
title = None
|
||||
for node in doctree.traverse(nodes.title):
|
||||
title = node.astext()
|
||||
break
|
||||
|
||||
if title:
|
||||
_manager.update_page_title(docname, title)
|
||||
|
||||
|
||||
def build_finished(app: Sphinx, exception):
|
||||
"""Called when the build is finished."""
|
||||
if exception is None:
|
||||
# Set the environment and master doc in the manager
|
||||
_manager.set_env(app.env)
|
||||
_manager.set_master_doc(app.config.master_doc)
|
||||
|
||||
# Set up configuration
|
||||
config = {
|
||||
"llms_txt_filename": app.config.llms_txt_filename,
|
||||
"llms_txt_verbose": app.config.llms_txt_verbose,
|
||||
}
|
||||
_manager.set_config(config)
|
||||
|
||||
# Get final titles from the environment at build completion
|
||||
if hasattr(app.env, "titles"):
|
||||
for docname, title_node in app.env.titles.items():
|
||||
if title_node:
|
||||
title = title_node.astext()
|
||||
_manager.update_page_title(docname, title)
|
||||
|
||||
# Create the combined file
|
||||
_manager.combine_sources(app.outdir, app.srcdir)
|
||||
|
||||
|
||||
def setup(app: Sphinx) -> Dict[str, Any]:
|
||||
"""Set up the Sphinx extension."""
|
||||
|
||||
# Add configuration options
|
||||
app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
|
||||
app.add_config_value("llms_txt_verbose", False, "env")
|
||||
|
||||
# Connect to Sphinx events
|
||||
app.connect("doctree-resolved", doctree_resolved)
|
||||
app.connect("build-finished", build_finished)
|
||||
|
||||
# Reset manager for each build
|
||||
global _manager
|
||||
_manager = LLMSFullManager()
|
||||
|
||||
return {
|
||||
"version": __version__,
|
||||
"parallel_read_safe": True,
|
||||
"parallel_write_safe": True,
|
||||
}
|
||||
Reference in New Issue
Block a user