Initial commit

.flake8

@@ -0,0 +1,12 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203, W503
+exclude = 
+    .git,
+    __pycache__,
+    .venv,
+    venv,
+    .eggs,
+    *.egg,
+    build,
+    dist

.gitignore

@@ -0,0 +1,21 @@
+*.pyc
+.idea/
+*.code-workspace
+
+# Unit test / coverage reports
+.tox
+
+# Distribution / packaging
+*.egg-info/
+dist/
+build/
+
+# Environments
+.venv
+
+# Sphinx documentation
+docs/_build/
+
+# vale packages
+docs/_vale/Microsoft/
+docs/_vale/write-good/

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+# Install pre-commit hooks via
+# pre-commit install
+
+repos:
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+    - id: isort
+
+  - repo: https://github.com/psf/black
+    rev: 24.10.0
+    hooks:
+    - id: black
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.1.1
+    hooks:
+    - id: flake8
+
+  - repo: https://github.com/sphinx-contrib/sphinx-lint
+    rev: v1.0.0
+    hooks:
+      - id: sphinx-lint
+        args: [--jobs=1]
+        files: ^docs/|CHANGELOG.rst|README.rst
+        types: [rst]

CHANGELOG.rst

@@ -0,0 +1,7 @@
+Changelog
+=========
+
+0.1.0
+-----
+
+- Initial release

MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE

README.md

@@ -0,0 +1,37 @@
+# Sphinx llms-full.txt Extension
+
+A Sphinx extension that creates a single combined documentation `llms-full.txt` file, written in reStructuredText.
+
+## Installation
+
+```bash
+pip install sphinx-llms-txt
+```
+
+## Usage
+
+1. Add the extension to your Sphinx configuration (`conf.py`):
+
+```python
+extensions = [
+    'sphinx_llms_txt',
+]
+```
+
+## Configuration Options
+
+### `llms_txt_filename`
+
+- **Type**: string
+- **Default**: `'llms-full.txt'`
+- **Description**: Name of the output file
+
+### `llms_txt_verbose`
+
+- **Type**: boolean  
+- **Default**: `False`
+- **Description**: Whether to include a summary in the build output
+
+## License
+
+MIT License - see LICENSE file for details.

pyproject.toml

@@ -0,0 +1,87 @@
+
+[build-system]
+requires = [
+    "setuptools",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sphinx-llms-txt"
+description = "llms-full.txt generator for Sphinx"
+authors = [
+    {name = "Jared Dillard", email = "jared.dillard@gmail.com"},
+]
+maintainers = [
+    {name = "Jared Dillard", email = "jared.dillard@gmail.com"},
+]
+classifiers = [
+    "Framework :: Sphinx :: Extension",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Documentation :: Sphinx",
+]
+license = {text = "MIT"}
+readme = "README.md"
+dynamic = ["version"]
+
+[project.urls]
+download = "https://pypi.org/project/sphinx-llms-txt/"
+source = "https://github.com/jdillard/sphinx-llms-txt"
+changelog = "https://github.com/jdillard/sphinx-llms-txt/blob/master/CHANGELOG.rst"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "black",
+    "flake8",
+    "mypy",
+    "isort",
+    "pre-commit",
+]
+test = [
+    "pytest>=7.0.0",
+]
+
+
+[tool.setuptools.dynamic]
+version = {attr = "sphinx_llms_txt.__version__"}
+
+[tool.black]
+line-length = 88
+target-version = ['py39', 'py310', 'py311', 'py312']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+
+[tool.mypy]
+python_version = "3.9"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_functions = ["test_*"]
+addopts = "-v"
+filterwarnings = [
+    "error",
+    "ignore::UserWarning",
+    "ignore::DeprecationWarning",
+]

sphinx_llms_txt/__init__.py

@@ -0,0 +1,295 @@
+"""
+Sphinx extension to create a combined sources file (llms-full.rst)
+that combines all documentation sources in the correct build order.
+"""
+
+from pathlib import Path
+from typing import Any, Dict, List
+
+from sphinx.application import Sphinx
+from sphinx.environment import BuildEnvironment
+from sphinx.util import logging
+
+__version__ = "0.1.0"
+
+logger = logging.getLogger(__name__)
+
+
+class LLMSFullManager:
+    """Manages the collection and ordering of documentation sources."""
+
+    def __init__(self):
+        self.page_titles: Dict[str, str] = {}
+        self.config: Dict[str, Any] = {}
+        self.master_doc: str = None
+        self.env: BuildEnvironment = None
+
+    def set_master_doc(self, master_doc: str):
+        """Set the master document name."""
+        self.master_doc = master_doc
+
+    def set_env(self, env: BuildEnvironment):
+        """Set the Sphinx environment."""
+        self.env = env
+
+    def update_page_title(self, docname: str, title: str):
+        """Update the title for a page."""
+        if title:
+            self.page_titles[docname] = title
+
+    def set_config(self, config: Dict[str, Any]):
+        """Set configuration options."""
+        self.config = config
+
+    def get_page_order(self) -> List[str]:
+        """Get the correct page order from the toctree structure."""
+        if not self.env or not self.master_doc:
+            return []
+
+        page_order = []
+        visited = set()
+
+        def collect_from_toctree(docname: str):
+            """Recursively collect documents from toctree."""
+            if docname in visited:
+                return
+
+            visited.add(docname)
+
+            # Add the current document
+            if docname not in page_order:
+                page_order.append(docname)
+
+            # Check for toctree entries in this document
+            try:
+                # Look for toctree_includes which contains the direct children
+                if (
+                    hasattr(self.env, "toctree_includes")
+                    and docname in self.env.toctree_includes
+                ):
+                    for child_docname in self.env.toctree_includes[docname]:
+                        collect_from_toctree(child_docname)
+                else:
+                    # Fallback: try to resolve and parse the toctree
+                    toctree = self.env.get_and_resolve_toctree(docname, None)
+                    if toctree:
+                        from docutils import nodes
+
+                        for node in toctree.traverse(nodes.reference):
+                            if "refuri" in node.attributes:
+                                refuri = node.attributes["refuri"]
+                                if refuri and refuri.endswith(".html"):
+                                    child_docname = refuri[:-5]  # Remove .html
+                                    if (
+                                        child_docname != docname
+                                    ):  # Avoid circular references
+                                        collect_from_toctree(child_docname)
+            except Exception as e:
+                logger.debug(f"Could not get toctree for {docname}: {e}")
+
+        # Start from the master document
+        collect_from_toctree(self.master_doc)
+
+        # Add any remaining documents not in the toctree (sorted)
+        if hasattr(self.env, "all_docs"):
+            remaining = sorted(
+                [doc for doc in self.env.all_docs.keys() if doc not in page_order]
+            )
+            page_order.extend(remaining)
+
+        return page_order
+
+    def combine_sources(self, outdir: str, srcdir: str):
+        """Combine all source files into a single file."""
+        # Get the correct page order
+        page_order = self.get_page_order()
+
+        if not page_order:
+            logger.warning(
+                "Could not determine page order, skipping llms-full creation"
+            )
+            return
+
+        # Determine output file name and location
+        output_filename = self.config.get("llms_txt_filename")
+        output_path = Path(outdir) / output_filename
+
+        # Find sources directory
+        sources_dir = None
+        possible_sources = [
+            Path(outdir) / "_sources",
+            Path(outdir) / "html" / "_sources",
+            Path(outdir) / "singlehtml" / "_sources",
+        ]
+
+        for path in possible_sources:
+            if path.exists():
+                sources_dir = path
+                break
+
+        if not sources_dir:
+            logger.warning(
+                "Could not find _sources directory, skipping llms-full creation"
+            )
+            return
+
+        # Collect all available source files
+        txt_files = {}
+        for f in sources_dir.glob("*.txt"):
+            txt_files[f.stem] = f
+
+        # Create a mapping from docnames to actual file names
+        docname_to_file = {}
+
+        # Try exact matches first
+        for docname in page_order:
+            if docname in txt_files:
+                docname_to_file[docname] = txt_files[docname]
+            else:
+                # Try with .rst extension
+                if f"{docname}.rst" in txt_files:
+                    docname_to_file[docname] = txt_files[f"{docname}.rst"]
+                # Try with .txt extension
+                elif f"{docname}.txt" in txt_files:
+                    docname_to_file[docname] = txt_files[f"{docname}.txt"]
+                # Try with underscores instead of hyphens
+                elif docname.replace("-", "_") in txt_files:
+                    docname_to_file[docname] = txt_files[docname.replace("-", "_")]
+                # Try with hyphens instead of underscores
+                elif docname.replace("_", "-") in txt_files:
+                    docname_to_file[docname] = txt_files[docname.replace("_", "-")]
+
+        # Generate content
+        content_parts = []
+
+        # Add pages in order
+        added_files = set()
+
+        for docname in page_order:
+            if docname in docname_to_file:
+                file_path = docname_to_file[docname]
+                content = self._read_source_file(file_path, docname)
+                if content:
+                    content_parts.append(content)
+                    added_files.add(file_path.stem)
+            else:
+                logger.warning(f"Source file not found for: {docname}")
+
+        # Add any remaining files (in alphabetical order)
+        remaining_files = sorted(
+            [name for name in txt_files if name not in added_files]
+        )
+        if remaining_files:
+            logger.info(f"Adding remaining files: {remaining_files}")
+        for file_stem in remaining_files:
+            file_path = txt_files[file_stem]
+            content = self._read_source_file(file_path, file_stem)
+            if content:
+                content_parts.append(content)
+
+        # Write combined file
+        try:
+            with open(output_path, "w", encoding="utf-8") as f:
+                f.write("\n".join(content_parts))
+
+            logger.info(
+                f"sphinx-llms-txt: created {output_path} with {len(txt_files)} sources"
+            )
+
+            # Log summary information if requested
+            if self.config.get("llms_txt_verbose"):
+                self._log_summary_info(page_order)
+
+        except Exception as e:
+            logger.error(f"Error writing combined sources file: {e}")
+
+    def _read_source_file(self, file_path: Path, docname: str) -> str:
+        """Read and format a single source file."""
+        try:
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            section_lines = [content, ""]
+
+            return "\n".join(section_lines)
+
+        except Exception as e:
+            logger.error(f"Error reading source file {file_path}: {e}")
+            return ""
+
+    def _log_summary_info(self, page_order: List[str]):
+        """Log summary information to the logger."""
+        logger.info("")
+        logger.info("llms-txt Summary")
+        logger.info("================")
+        logger.info(f"Total pages: {len(page_order)}")
+        logger.info(f"Configuration: {self.config}")
+        logger.info("Page order:")
+        for i, docname in enumerate(page_order, 1):
+            title = self.page_titles.get(docname, docname)
+            logger.info(f"{i:3d}. {docname} - {title}")
+
+
+# Global manager instance
+_manager = LLMSFullManager()
+
+
+def doctree_resolved(app: Sphinx, doctree, docname: str):
+    """Called when a docname has been resolved to a document."""
+    # Extract title from the document
+    from docutils import nodes
+
+    title = None
+    for node in doctree.traverse(nodes.title):
+        title = node.astext()
+        break
+
+    if title:
+        _manager.update_page_title(docname, title)
+
+
+def build_finished(app: Sphinx, exception):
+    """Called when the build is finished."""
+    if exception is None:
+        # Set the environment and master doc in the manager
+        _manager.set_env(app.env)
+        _manager.set_master_doc(app.config.master_doc)
+
+        # Set up configuration
+        config = {
+            "llms_txt_filename": app.config.llms_txt_filename,
+            "llms_txt_verbose": app.config.llms_txt_verbose,
+        }
+        _manager.set_config(config)
+
+        # Get final titles from the environment at build completion
+        if hasattr(app.env, "titles"):
+            for docname, title_node in app.env.titles.items():
+                if title_node:
+                    title = title_node.astext()
+                    _manager.update_page_title(docname, title)
+
+        # Create the combined file
+        _manager.combine_sources(app.outdir, app.srcdir)
+
+
+def setup(app: Sphinx) -> Dict[str, Any]:
+    """Set up the Sphinx extension."""
+
+    # Add configuration options
+    app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
+    app.add_config_value("llms_txt_verbose", False, "env")
+
+    # Connect to Sphinx events
+    app.connect("doctree-resolved", doctree_resolved)
+    app.connect("build-finished", build_finished)
+
+    # Reset manager for each build
+    global _manager
+    _manager = LLMSFullManager()
+
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }