From 2ca3052a2c1d1757b92fc679e6f25be276926aa4 Mon Sep 17 00:00:00 2001 From: Jared Dillard Date: Fri, 16 May 2025 14:33:34 -0700 Subject: [PATCH] Automatically add content from `.. include::` directives (#6) --- CHANGELOG.rst | 1 + README.md | 5 ++ sphinx_llms_txt/__init__.py | 98 +++++++++++++++++++++++++ tests/CHANGELOG.rst | 13 ++++ tests/roots/basic/CHANGELOG.rst | 13 ++++ tests/roots/basic/index.rst | 1 + tests/roots/basic/page_with_include.rst | 8 ++ tests/test_integration.py | 8 ++ tests/test_llms_txt.py | 94 ++++++++++++++++++++++++ 9 files changed, 241 insertions(+) create mode 100644 tests/CHANGELOG.rst create mode 100644 tests/roots/basic/CHANGELOG.rst create mode 100644 tests/roots/basic/page_with_include.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 51afeb5..a710cdc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ Changelog ----- - Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size +- Automatically add content from `.. include::` directives 0.1.0 ----- diff --git a/README.md b/README.md index 66cefde..2ee5259 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,11 @@ extensions = [ - **Default**: `None` (no limit) - **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes. +## Features + +- Automatically add content from `.. include::` directives + + ## License MIT License - see LICENSE file for details. diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index df2fb97..b61133e 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -100,6 +100,10 @@ class LLMSFullManager: def combine_sources(self, outdir: str, srcdir: str): """Combine all source files into a single file.""" + # Store the source directory for resolving include directives + self.srcdir = srcdir + self.outdir = outdir + # Get the correct page order page_order = self.get_page_order() @@ -240,6 +244,9 @@ class LLMSFullManager: def _read_source_file(self, file_path: Path, docname: str) -> tuple: """Read and format a single source file. + Handles include directives by replacing them with the content of the included + file. + Returns: tuple: (content_str, line_count) where line_count is the number of lines in the file @@ -248,6 +255,9 @@ class LLMSFullManager: with open(file_path, "r", encoding="utf-8") as f: content = f.read() + # Process include directives + content = self._process_includes(content, file_path) + # Count the lines in the content line_count = content.count("\n") + (0 if content.endswith("\n") else 1) @@ -261,6 +271,94 @@ class LLMSFullManager: logger.error(f"sphinx-llm-txt: Error reading source file {file_path}: {e}") return "", 0 + def _process_includes(self, content: str, source_path: Path) -> str: + """Process include directives in content. + + Args: + content: The source content to process + source_path: Path to the source file (to resolve relative paths) + + Returns: + Processed content with include directives replaced with included content + """ + import os + import re + + # Find all include directives using regex + include_pattern = re.compile(r"^\.\.\s+include::\s+([^\s]+)\s*$", re.MULTILINE) + + # Function to replace each include with content + def replace_include(match): + include_path = match.group(1) + + # Try multiple possible paths for the include file + possible_paths = [] + + # If it's an absolute path, use it directly + if os.path.isabs(include_path): + possible_paths.append(Path(include_path)) + else: + # Relative to the source file (in _sources directory) + possible_paths.append((source_path.parent / include_path).resolve()) + + # If we're in _sources directory, try relative to the original source + # directory + if "_sources" in str(source_path): + # Extract the relative path portion from the source path + rel_path = None + try: + # Get the part after _sources/ + path_parts = str(source_path).split("_sources/") + if len(path_parts) > 1: + rel_path = path_parts[1] + # Remove .txt extension if present + if rel_path.endswith(".txt"): + rel_path = rel_path[:-4] + except Exception: + pass + + # If we have the original source directory from Sphinx + if hasattr(self, "srcdir") and self.srcdir: + # Try in the srcdir root + possible_paths.append( + (Path(self.srcdir) / include_path).resolve() + ) + + # If we have a relative path, try in the corresponding source + # subdirectory + if rel_path: + rel_dir = os.path.dirname(rel_path) + if rel_dir: + possible_paths.append( + ( + Path(self.srcdir) / rel_dir / include_path + ).resolve() + ) + + # Try each possible path + for path_to_try in possible_paths: + try: + if path_to_try.exists(): + with open(path_to_try, "r", encoding="utf-8") as f: + included_content = f.read() + return included_content + except Exception as e: + logger.error( + f"sphinx-llms-txt: Error reading include file {path_to_try}:" + f" {e}" + ) + continue + + # If we get here, we couldn't find the file + paths_tried = ", ".join(str(p) for p in possible_paths) + logger.warning(f"sphinx-llms-txt: Include file not found: {include_path}") + logger.debug(f"sphinx-llms-txt: Tried paths: {paths_tried}") + return f"[Include file not found: {include_path}]" + + # Replace all includes with their content + processed_content = include_pattern.sub(replace_include, content) + return processed_content + def _log_summary_info(self, page_order: List[str], total_line_count: int = 0): """Log summary information to the logger.""" logger.info("") diff --git a/tests/CHANGELOG.rst b/tests/CHANGELOG.rst new file mode 100644 index 0000000..340d843 --- /dev/null +++ b/tests/CHANGELOG.rst @@ -0,0 +1,13 @@ +Changelog +========= + +0.2.0 (2025-01-01) +------------------ + +* Added include directive processing feature +* Added maxlines configuration + +0.1.0 (2024-01-01) +------------------ + +* Initial release diff --git a/tests/roots/basic/CHANGELOG.rst b/tests/roots/basic/CHANGELOG.rst new file mode 100644 index 0000000..340d843 --- /dev/null +++ b/tests/roots/basic/CHANGELOG.rst @@ -0,0 +1,13 @@ +Changelog +========= + +0.2.0 (2025-01-01) +------------------ + +* Added include directive processing feature +* Added maxlines configuration + +0.1.0 (2024-01-01) +------------------ + +* Initial release diff --git a/tests/roots/basic/index.rst b/tests/roots/basic/index.rst index 1c63d95..0893a95 100644 --- a/tests/roots/basic/index.rst +++ b/tests/roots/basic/index.rst @@ -7,6 +7,7 @@ Welcome to Test Project's documentation! page1 page2 + page_with_include Indices and tables ================== diff --git a/tests/roots/basic/page_with_include.rst b/tests/roots/basic/page_with_include.rst new file mode 100644 index 0000000..3945616 --- /dev/null +++ b/tests/roots/basic/page_with_include.rst @@ -0,0 +1,8 @@ +Page With Include +=============== + +This is a test page that includes another file: + +.. include:: CHANGELOG.rst + +This content comes after the include. \ No newline at end of file diff --git a/tests/test_integration.py b/tests/test_integration.py index a06ea78..79abe9e 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -25,6 +25,14 @@ def test_build_html_with_llms_txt(basic_sphinx_app): assert "Content for section 1" in content assert "Content for section A" in content + # Check that the include directive has been processed + assert "Page With Include" in content + assert "This is a test page that includes another file:" in content + assert "Changelog" in content # Content from the included file + assert "0.2.0 (2025-01-01)" in content # Content from the included file + assert "0.1.0 (2024-01-01)" in content # Additional content from the included file + assert "This content comes after the include." in content + def test_custom_filename(temp_dir, rootdir): """Test using a custom filename for the output.""" diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index 91bb2cd..a781612 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -81,3 +81,97 @@ def test_empty_page_order(): # Set only master_doc, but not env manager.set_master_doc("index") assert manager.get_page_order() == [] + + +def test_process_includes(tmp_path): + """Test that include directives are processed correctly.""" + # Create a manager + manager = LLMSFullManager() + + # Create a test file with an include directive + include_content = "This is included content.\nWith multiple lines." + include_file = tmp_path / "included.txt" + with open(include_file, "w", encoding="utf-8") as f: + f.write(include_content) + + # Create a source file that includes the test file + source_content = ( + "Line before include.\n.. include:: included.txt\nLine after include." + ) + source_file = tmp_path / "source.txt" + with open(source_file, "w", encoding="utf-8") as f: + f.write(source_content) + + # Process the include directive + processed_content = manager._process_includes(source_content, source_file) + + # Check that the include directive was replaced with the content + expected_content = ( + "Line before include.\nThis is included content.\nWith multiple" + " lines.\nLine after include." + ) + assert processed_content == expected_content + + +def test_process_includes_with_relative_paths(tmp_path): + """Test that include directives with relative paths are processed correctly.""" + # Create a manager + manager = LLMSFullManager() + + # Set up a more complex directory structure + docs_dir = tmp_path / "docs" + docs_dir.mkdir() + + # Create the original source directory structure + source_dir = docs_dir / "source" + source_dir.mkdir() + + # Create a subdirectory + subdir = source_dir / "subdir" + subdir.mkdir() + + # Create an includes directory + includes_dir = source_dir / "includes" + includes_dir.mkdir() + + # Set the srcdir on the manager + manager.srcdir = str(source_dir) + + # Create the included file in the includes directory + include_content = "This is included content from another directory." + include_file = includes_dir / "common.txt" + with open(include_file, "w", encoding="utf-8") as f: + f.write(include_content) + + # Create a source file in the subdirectory that includes the file from includes + source_content = ( + "Line before include.\n.. include:: ../includes/common.txt\nLine after include." + ) + source_file = subdir / "page.txt" + with open(source_file, "w", encoding="utf-8") as f: + f.write(source_content) + + # Create the _sources directory to mimic Sphinx build output + build_dir = tmp_path / "build" + build_dir.mkdir() + sources_dir = build_dir / "_sources" + sources_dir.mkdir() + + # Create the same structure in the _sources directory + sources_subdir = sources_dir / "subdir" + sources_subdir.mkdir() + + # Copy the source file to the _sources directory + sources_file = sources_subdir / "page.txt" + with open(sources_file, "w", encoding="utf-8") as f: + f.write(source_content) + + # Process the include directive from the _sources file + processed_content = manager._process_includes(source_content, sources_file) + + # Check that the include directive was replaced with the content + expected_content = ( + "Line before include.\nThis is included content from another" + " directory.\nLine after include." + ) + assert processed_content == expected_content