Automatically add content from .. include:: directives (#6)

CHANGELOG.rst

@@ -5,6 +5,7 @@ Changelog
 -----
 
 - Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size
+- Automatically add content from `.. include::` directives
 
 0.1.0
 -----

README.md

@@ -38,6 +38,11 @@ extensions = [
 - **Default**: `None` (no limit)
 - **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes.
 
+## Features
+
+- Automatically add content from `.. include::` directives
+
+
 ## License
 
 MIT License - see LICENSE file for details.

sphinx_llms_txt/__init__.py

@@ -100,6 +100,10 @@ class LLMSFullManager:
 
     def combine_sources(self, outdir: str, srcdir: str):
         """Combine all source files into a single file."""
+        # Store the source directory for resolving include directives
+        self.srcdir = srcdir
+        self.outdir = outdir
+
         # Get the correct page order
         page_order = self.get_page_order()
 
@@ -240,6 +244,9 @@ class LLMSFullManager:
     def _read_source_file(self, file_path: Path, docname: str) -> tuple:
         """Read and format a single source file.
 
+        Handles include directives by replacing them with the content of the included
+        file.
+
         Returns:
             tuple: (content_str, line_count) where line_count is the number of lines
                    in the file
@@ -248,6 +255,9 @@ class LLMSFullManager:
             with open(file_path, "r", encoding="utf-8") as f:
                 content = f.read()
 
+            # Process include directives
+            content = self._process_includes(content, file_path)
+
             # Count the lines in the content
             line_count = content.count("\n") + (0 if content.endswith("\n") else 1)
 
@@ -261,6 +271,94 @@ class LLMSFullManager:
             logger.error(f"sphinx-llm-txt: Error reading source file {file_path}: {e}")
             return "", 0
 
+    def _process_includes(self, content: str, source_path: Path) -> str:
+        """Process include directives in content.
+
+        Args:
+            content: The source content to process
+            source_path: Path to the source file (to resolve relative paths)
+
+        Returns:
+            Processed content with include directives replaced with included content
+        """
+        import os
+        import re
+
+        # Find all include directives using regex
+        include_pattern = re.compile(r"^\.\.\s+include::\s+([^\s]+)\s*$", re.MULTILINE)
+
+        # Function to replace each include with content
+        def replace_include(match):
+            include_path = match.group(1)
+
+            # Try multiple possible paths for the include file
+            possible_paths = []
+
+            # If it's an absolute path, use it directly
+            if os.path.isabs(include_path):
+                possible_paths.append(Path(include_path))
+            else:
+                # Relative to the source file (in _sources directory)
+                possible_paths.append((source_path.parent / include_path).resolve())
+
+                # If we're in _sources directory, try relative to the original source
+                # directory
+                if "_sources" in str(source_path):
+                    # Extract the relative path portion from the source path
+                    rel_path = None
+                    try:
+                        # Get the part after _sources/
+                        path_parts = str(source_path).split("_sources/")
+                        if len(path_parts) > 1:
+                            rel_path = path_parts[1]
+                            # Remove .txt extension if present
+                            if rel_path.endswith(".txt"):
+                                rel_path = rel_path[:-4]
+                    except Exception:
+                        pass
+
+                    # If we have the original source directory from Sphinx
+                    if hasattr(self, "srcdir") and self.srcdir:
+                        # Try in the srcdir root
+                        possible_paths.append(
+                            (Path(self.srcdir) / include_path).resolve()
+                        )
+
+                        # If we have a relative path, try in the corresponding source
+                        # subdirectory
+                        if rel_path:
+                            rel_dir = os.path.dirname(rel_path)
+                            if rel_dir:
+                                possible_paths.append(
+                                    (
+                                        Path(self.srcdir) / rel_dir / include_path
+                                    ).resolve()
+                                )
+
+            # Try each possible path
+            for path_to_try in possible_paths:
+                try:
+                    if path_to_try.exists():
+                        with open(path_to_try, "r", encoding="utf-8") as f:
+                            included_content = f.read()
+                        return included_content
+                except Exception as e:
+                    logger.error(
+                        f"sphinx-llms-txt: Error reading include file {path_to_try}:"
+                        f" {e}"
+                    )
+                    continue
+
+            # If we get here, we couldn't find the file
+            paths_tried = ", ".join(str(p) for p in possible_paths)
+            logger.warning(f"sphinx-llms-txt: Include file not found: {include_path}")
+            logger.debug(f"sphinx-llms-txt: Tried paths: {paths_tried}")
+            return f"[Include file not found: {include_path}]"
+
+        # Replace all includes with their content
+        processed_content = include_pattern.sub(replace_include, content)
+        return processed_content
+
     def _log_summary_info(self, page_order: List[str], total_line_count: int = 0):
         """Log summary information to the logger."""
         logger.info("")

tests/CHANGELOG.rst

@@ -0,0 +1,13 @@
+Changelog
+=========
+
+0.2.0 (2025-01-01)
+------------------
+
+* Added include directive processing feature
+* Added maxlines configuration
+
+0.1.0 (2024-01-01)
+------------------
+
+* Initial release

tests/roots/basic/CHANGELOG.rst

@@ -0,0 +1,13 @@
+Changelog
+=========
+
+0.2.0 (2025-01-01)
+------------------
+
+* Added include directive processing feature
+* Added maxlines configuration
+
+0.1.0 (2024-01-01)
+------------------
+
+* Initial release

tests/roots/basic/index.rst

@@ -7,6 +7,7 @@ Welcome to Test Project's documentation!
 
    page1
    page2
+   page_with_include
 
 Indices and tables
 ==================

tests/roots/basic/page_with_include.rst

@@ -0,0 +1,8 @@
+Page With Include
+===============
+
+This is a test page that includes another file:
+
+.. include:: CHANGELOG.rst
+
+This content comes after the include.

tests/test_integration.py

@@ -25,6 +25,14 @@ def test_build_html_with_llms_txt(basic_sphinx_app):
     assert "Content for section 1" in content
     assert "Content for section A" in content
 
+    # Check that the include directive has been processed
+    assert "Page With Include" in content
+    assert "This is a test page that includes another file:" in content
+    assert "Changelog" in content  # Content from the included file
+    assert "0.2.0 (2025-01-01)" in content  # Content from the included file
+    assert "0.1.0 (2024-01-01)" in content  # Additional content from the included file
+    assert "This content comes after the include." in content
+
 
 def test_custom_filename(temp_dir, rootdir):
     """Test using a custom filename for the output."""

tests/test_llms_txt.py

@@ -81,3 +81,97 @@ def test_empty_page_order():
     # Set only master_doc, but not env
     manager.set_master_doc("index")
     assert manager.get_page_order() == []
+
+
+def test_process_includes(tmp_path):
+    """Test that include directives are processed correctly."""
+    # Create a manager
+    manager = LLMSFullManager()
+
+    # Create a test file with an include directive
+    include_content = "This is included content.\nWith multiple lines."
+    include_file = tmp_path / "included.txt"
+    with open(include_file, "w", encoding="utf-8") as f:
+        f.write(include_content)
+
+    # Create a source file that includes the test file
+    source_content = (
+        "Line before include.\n.. include:: included.txt\nLine after include."
+    )
+    source_file = tmp_path / "source.txt"
+    with open(source_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Process the include directive
+    processed_content = manager._process_includes(source_content, source_file)
+
+    # Check that the include directive was replaced with the content
+    expected_content = (
+        "Line before include.\nThis is included content.\nWith multiple"
+        " lines.\nLine after include."
+    )
+    assert processed_content == expected_content
+
+
+def test_process_includes_with_relative_paths(tmp_path):
+    """Test that include directives with relative paths are processed correctly."""
+    # Create a manager
+    manager = LLMSFullManager()
+
+    # Set up a more complex directory structure
+    docs_dir = tmp_path / "docs"
+    docs_dir.mkdir()
+
+    # Create the original source directory structure
+    source_dir = docs_dir / "source"
+    source_dir.mkdir()
+
+    # Create a subdirectory
+    subdir = source_dir / "subdir"
+    subdir.mkdir()
+
+    # Create an includes directory
+    includes_dir = source_dir / "includes"
+    includes_dir.mkdir()
+
+    # Set the srcdir on the manager
+    manager.srcdir = str(source_dir)
+
+    # Create the included file in the includes directory
+    include_content = "This is included content from another directory."
+    include_file = includes_dir / "common.txt"
+    with open(include_file, "w", encoding="utf-8") as f:
+        f.write(include_content)
+
+    # Create a source file in the subdirectory that includes the file from includes
+    source_content = (
+        "Line before include.\n.. include:: ../includes/common.txt\nLine after include."
+    )
+    source_file = subdir / "page.txt"
+    with open(source_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Create the _sources directory to mimic Sphinx build output
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Create the same structure in the _sources directory
+    sources_subdir = sources_dir / "subdir"
+    sources_subdir.mkdir()
+
+    # Copy the source file to the _sources directory
+    sources_file = sources_subdir / "page.txt"
+    with open(sources_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Process the include directive from the _sources file
+    processed_content = manager._process_includes(source_content, sources_file)
+
+    # Check that the include directive was replaced with the content
+    expected_content = (
+        "Line before include.\nThis is included content from another"
+        " directory.\nLine after include."
+    )
+    assert processed_content == expected_content