Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9bfdc38b60 | ||
|
|
10af26acfa | ||
|
|
fcfc9c6f70 | ||
|
|
b397cce765 | ||
|
|
af34e0f293 |
@@ -128,8 +128,11 @@ class DocumentProcessor:
|
|||||||
Returns:
|
Returns:
|
||||||
Processed content with directive paths properly resolved
|
Processed content with directive paths properly resolved
|
||||||
"""
|
"""
|
||||||
|
# Get code block ranges to skip directives inside them
|
||||||
|
code_block_ranges = self._get_code_block_ranges(content)
|
||||||
|
|
||||||
# Get the configured path directives to process
|
# Get the configured path directives to process
|
||||||
default_path_directives = ["image", "figure"]
|
default_path_directives = ["image", "figure", "literalinclude"]
|
||||||
custom_path_directives = self.config.get("llms_txt_directives")
|
custom_path_directives = self.config.get("llms_txt_directives")
|
||||||
path_directives = set(default_path_directives + custom_path_directives)
|
path_directives = set(default_path_directives + custom_path_directives)
|
||||||
|
|
||||||
@@ -143,6 +146,11 @@ class DocumentProcessor:
|
|||||||
is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
|
is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
|
||||||
|
|
||||||
def replace_directive_path(match, base_url=base_url, is_test=is_test):
|
def replace_directive_path(match, base_url=base_url, is_test=is_test):
|
||||||
|
# Check if this directive is within a code block
|
||||||
|
if self._is_in_code_block(match.start(), code_block_ranges):
|
||||||
|
# This directive is inside a code block, don't process it
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
prefix = match.group(1) # The entire directive prefix including whitespace
|
prefix = match.group(1) # The entire directive prefix including whitespace
|
||||||
path = match.group(3).strip() # The path argument
|
path = match.group(3).strip() # The path argument
|
||||||
|
|
||||||
@@ -276,6 +284,71 @@ class DocumentProcessor:
|
|||||||
|
|
||||||
return possible_paths
|
return possible_paths
|
||||||
|
|
||||||
|
def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
|
||||||
|
"""Find all code block ranges in the content.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content: The source content to analyze
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (start, end) tuples representing code block character
|
||||||
|
ranges
|
||||||
|
"""
|
||||||
|
code_block_ranges = []
|
||||||
|
|
||||||
|
# Match code block as well as `code` and `sourcecode` aliases
|
||||||
|
code_block_pattern = re.compile(
|
||||||
|
r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
|
||||||
|
)
|
||||||
|
|
||||||
|
for match in code_block_pattern.finditer(content):
|
||||||
|
start_pos = match.start()
|
||||||
|
indent = match.group(1)
|
||||||
|
indent_len = len(indent)
|
||||||
|
|
||||||
|
# Find the end of the code block by looking for the next line
|
||||||
|
# that is not indented more than the directive
|
||||||
|
block_start = match.end()
|
||||||
|
pos = block_start
|
||||||
|
|
||||||
|
# Skip any blank lines immediately after the directive
|
||||||
|
while pos < len(content) and content[pos] in "\n":
|
||||||
|
pos += 1
|
||||||
|
|
||||||
|
# Find where the code block ends
|
||||||
|
lines = content[pos:].split("\n")
|
||||||
|
block_end = pos
|
||||||
|
for line in lines:
|
||||||
|
if line.strip(): # Non-empty line
|
||||||
|
# Check indentation level
|
||||||
|
line_indent = len(line) - len(line.lstrip())
|
||||||
|
if line_indent <= indent_len:
|
||||||
|
# The block ends when we find a line that is indented
|
||||||
|
# less than the directive itself
|
||||||
|
break
|
||||||
|
block_end += len(line) + 1 # +1 for the newline
|
||||||
|
|
||||||
|
code_block_ranges.append((start_pos, block_end))
|
||||||
|
|
||||||
|
return code_block_ranges
|
||||||
|
|
||||||
|
def _is_in_code_block(
|
||||||
|
self, match_start: int, code_block_ranges: List[Tuple[int, int]]
|
||||||
|
) -> bool:
|
||||||
|
"""Check if a match position is within a code block.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
match_start: The starting position of the match
|
||||||
|
code_block_ranges: List of (start, end) tuples for code blocks
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the match is within a code block, False otherwise
|
||||||
|
"""
|
||||||
|
for block_start, block_end in code_block_ranges:
|
||||||
|
if block_start <= match_start < block_end:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _process_includes(self, content: str, source_path: Path) -> str:
|
def _process_includes(self, content: str, source_path: Path) -> str:
|
||||||
"""Process include directives in content.
|
"""Process include directives in content.
|
||||||
|
|
||||||
@@ -286,11 +359,18 @@ class DocumentProcessor:
|
|||||||
Returns:
|
Returns:
|
||||||
Processed content with include directives replaced with included content
|
Processed content with include directives replaced with included content
|
||||||
"""
|
"""
|
||||||
|
code_block_ranges = self._get_code_block_ranges(content)
|
||||||
|
|
||||||
# Find all include directives using regex
|
# Find all include directives using regex
|
||||||
include_pattern = build_directive_pattern(["include"])
|
include_pattern = build_directive_pattern(["include"])
|
||||||
|
|
||||||
# Function to replace each include with content
|
# Function to replace each include with content
|
||||||
def replace_include(match):
|
def replace_include(match):
|
||||||
|
# Check if this include is within a code block
|
||||||
|
if self._is_in_code_block(match.start(), code_block_ranges):
|
||||||
|
# This include is inside a code block, don't process it
|
||||||
|
return match.group(0)
|
||||||
|
|
||||||
include_path = match.group(3)
|
include_path = match.group(3)
|
||||||
directive_part = match.group(
|
directive_part = match.group(
|
||||||
1
|
1
|
||||||
|
|||||||
@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
|
|||||||
assert processed_content == expected_content
|
assert processed_content == expected_content
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_includes_in_code_block(tmp_path):
|
||||||
|
"""Test that an `include` within a `code-block` is not processed."""
|
||||||
|
# Create a processor
|
||||||
|
config = {"llms_txt_directives": []}
|
||||||
|
processor = DocumentProcessor(config)
|
||||||
|
|
||||||
|
# Create a source file that uses include syntax within a `code-block`
|
||||||
|
source_content = (
|
||||||
|
"Normal paragraph.\n\n"
|
||||||
|
".. code-block:: rst\n\n"
|
||||||
|
" .. include:: foo.txt\n\n"
|
||||||
|
"Another normal paragraph."
|
||||||
|
)
|
||||||
|
source_file = tmp_path / "source.txt"
|
||||||
|
with open(source_file, "w", encoding="utf-8") as f:
|
||||||
|
f.write(source_content)
|
||||||
|
|
||||||
|
# Run the include directive processor
|
||||||
|
processed_content = processor._process_includes(source_content, source_file)
|
||||||
|
|
||||||
|
# Check that the include directive was not processed
|
||||||
|
expected_content = source_content
|
||||||
|
assert processed_content == expected_content
|
||||||
|
|
||||||
|
|
||||||
def test_process_includes_with_relative_paths(tmp_path):
|
def test_process_includes_with_relative_paths(tmp_path):
|
||||||
"""Test that include directives with relative paths are processed correctly."""
|
"""Test that include directives with relative paths are processed correctly."""
|
||||||
# Create a processor
|
# Create a processor
|
||||||
|
|||||||
Reference in New Issue
Block a user