From 1dd8119cfd6888292e2f42c723c93a3278f485e7 Mon Sep 17 00:00:00 2001 From: Kayce Basques Date: Tue, 16 Dec 2025 14:30:38 -0800 Subject: [PATCH] Don't process includes within code blocks (#58) --- CHANGELOG.rst | 5 +++ sphinx_llms_txt/__init__.py | 2 +- sphinx_llms_txt/processor.py | 82 +++++++++++++++++++++++++++++++++++- tests/test_llms_txt.py | 25 +++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 49a73c3..83059ae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Changelog ========= +0.7.1 +----- + +- Don't process includes within code blocks + 0.7.0 ----- diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index b627682..c64762b 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -21,7 +21,7 @@ from .manager import LLMSFullManager from .processor import DocumentProcessor from .writer import FileWriter -__version__ = "0.7.0" +__version__ = "0.7.1" # Export classes needed by tests __all__ = [ diff --git a/sphinx_llms_txt/processor.py b/sphinx_llms_txt/processor.py index f04ddfd..4ca9427 100644 --- a/sphinx_llms_txt/processor.py +++ b/sphinx_llms_txt/processor.py @@ -128,8 +128,11 @@ class DocumentProcessor: Returns: Processed content with directive paths properly resolved """ + # Get code block ranges to skip directives inside them + code_block_ranges = self._get_code_block_ranges(content) + # Get the configured path directives to process - default_path_directives = ["image", "figure"] + default_path_directives = ["image", "figure", "literalinclude"] custom_path_directives = self.config.get("llms_txt_directives") path_directives = set(default_path_directives + custom_path_directives) @@ -143,6 +146,11 @@ class DocumentProcessor: is_test = "pytest" in str(source_path) and "subdir" in str(source_path) def replace_directive_path(match, base_url=base_url, is_test=is_test): + # Check if this directive is within a code block + if self._is_in_code_block(match.start(), code_block_ranges): + # This directive is inside a code block, don't process it + return match.group(0) + prefix = match.group(1) # The entire directive prefix including whitespace path = match.group(3).strip() # The path argument @@ -276,6 +284,71 @@ class DocumentProcessor: return possible_paths + def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]: + """Find all code block ranges in the content. + + Args: + content: The source content to analyze + + Returns: + List of (start, end) tuples representing code block character + ranges + """ + code_block_ranges = [] + + # Match code block as well as `code` and `sourcecode` aliases + code_block_pattern = re.compile( + r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE + ) + + for match in code_block_pattern.finditer(content): + start_pos = match.start() + indent = match.group(1) + indent_len = len(indent) + + # Find the end of the code block by looking for the next line + # that is not indented more than the directive + block_start = match.end() + pos = block_start + + # Skip any blank lines immediately after the directive + while pos < len(content) and content[pos] in "\n": + pos += 1 + + # Find where the code block ends + lines = content[pos:].split("\n") + block_end = pos + for line in lines: + if line.strip(): # Non-empty line + # Check indentation level + line_indent = len(line) - len(line.lstrip()) + if line_indent <= indent_len: + # The block ends when we find a line that is indented + # less than the directive itself + break + block_end += len(line) + 1 # +1 for the newline + + code_block_ranges.append((start_pos, block_end)) + + return code_block_ranges + + def _is_in_code_block( + self, match_start: int, code_block_ranges: List[Tuple[int, int]] + ) -> bool: + """Check if a match position is within a code block. + + Args: + match_start: The starting position of the match + code_block_ranges: List of (start, end) tuples for code blocks + + Returns: + True if the match is within a code block, False otherwise + """ + for block_start, block_end in code_block_ranges: + if block_start <= match_start < block_end: + return True + return False + def _process_includes(self, content: str, source_path: Path) -> str: """Process include directives in content. @@ -286,11 +359,18 @@ class DocumentProcessor: Returns: Processed content with include directives replaced with included content """ + code_block_ranges = self._get_code_block_ranges(content) + # Find all include directives using regex include_pattern = build_directive_pattern(["include"]) # Function to replace each include with content def replace_include(match): + # Check if this include is within a code block + if self._is_in_code_block(match.start(), code_block_ranges): + # This include is inside a code block, don't process it + return match.group(0) + include_path = match.group(3) directive_part = match.group( 1 diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index 412da2c..789b358 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -198,6 +198,31 @@ def test_process_includes(tmp_path): assert processed_content == expected_content +def test_process_includes_in_code_block(tmp_path): + """Test that an `include` within a `code-block` is not processed.""" + # Create a processor + config = {"llms_txt_directives": []} + processor = DocumentProcessor(config) + + # Create a source file that uses include syntax within a `code-block` + source_content = ( + "Normal paragraph.\n\n" + ".. code-block:: rst\n\n" + " .. include:: foo.txt\n\n" + "Another normal paragraph." + ) + source_file = tmp_path / "source.txt" + with open(source_file, "w", encoding="utf-8") as f: + f.write(source_content) + + # Run the include directive processor + processed_content = processor._process_includes(source_content, source_file) + + # Check that the include directive was not processed + expected_content = source_content + assert processed_content == expected_content + + def test_process_includes_with_relative_paths(tmp_path): """Test that include directives with relative paths are processed correctly.""" # Create a processor