Xarkam/sphinx-llms-txt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add source code files to llms-full.txt (#24)

Browse Source
This commit is contained in:
Jared Dillard
2025-08-04 18:22:57 -07:00
committed by GitHub
parent 368c349d58
commit 0ee28290db
7 changed files with 714 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sphinx_llms_txt/manager.py
+445 -2
View File
@@ -2,8 +2,10 @@
Main manager module for sphinx-llms-txt.
"""
import glob
import subprocess
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
@@ -16,6 +18,104 @@ from .writer import FileWriter
logger = logging.getLogger(__name__)
def _get_git_root(path: Path) -> Optional[Path]:
"""Get the git root directory for a given path."""
try:
result = subprocess.run(
["git", "rev-parse", "--show-toplevel"],
cwd=path,
capture_output=True,
text=True,
check=True,
)
return Path(result.stdout.strip())
except (subprocess.CalledProcessError, FileNotFoundError):
return None
def _get_language_from_extension(file_path: Path) -> str:
"""Map file extension to language identifier for code blocks."""
extension_map = {
".py": "python",
".js": "javascript",
".jsx": "jsx",
".ts": "typescript",
".tsx": "tsx",
".java": "java",
".c": "c",
".cpp": "cpp",
".cc": "cpp",
".cxx": "cpp",
".h": "c",
".hpp": "cpp",
".cs": "csharp",
".php": "php",
".rb": "ruby",
".go": "go",
".rs": "rust",
".swift": "swift",
".kt": "kotlin",
".scala": "scala",
".sh": "bash",
".bash": "bash",
".zsh": "zsh",
".fish": "fish",
".ps1": "powershell",
".html": "html",
".htm": "html",
".xml": "xml",
".css": "css",
".scss": "scss",
".sass": "sass",
".less": "less",
".json": "json",
".yaml": "yaml",
".yml": "yaml",
".toml": "toml",
".ini": "ini",
".cfg": "ini",
".conf": "ini",
".sql": "sql",
".md": "markdown",
".rst": "rst",
".txt": "text",
".dockerfile": "dockerfile",
".dockerignore": "text",
".gitignore": "text",
".gitattributes": "text",
".editorconfig": "ini",
".makefile": "makefile",
".r": "r",
".R": "r",
".m": "matlab",
".pl": "perl",
".lua": "lua",
".vim": "vim",
".vimrc": "vim",
".proto": "protobuf",
".thrift": "thrift",
".graphql": "graphql",
".gql": "graphql",
}
# Get the extension from the file path
ext = file_path.suffix.lower()
# Handle special cases like Makefile, Dockerfile without extension
if not ext:
name = file_path.name.lower()
if name in ["makefile", "gnumakefile"]:
return "makefile"
elif name in ["dockerfile", "dockerfile.dev", "dockerfile.prod"]:
return "dockerfile"
elif name.startswith("dockerfile."):
return "dockerfile"
else:
return "text"
return extension_map.get(ext, "text")
class LLMSFullManager:
"""Manages the collection and ordering of documentation sources."""
@@ -163,9 +263,15 @@ class LLMSFullManager:
# Generate content
content_parts = []
# Track code files for later processing
code_file_parts = []
# Count lines in code files (initially 0)
code_files_line_count = 0
# Add pages in order
added_files = set()
total_line_count = 0
total_line_count = code_files_line_count
max_lines = self.config.get("llms_txt_full_max_size")
abort_due_to_max_lines = False
@@ -277,6 +383,37 @@ class LLMSFullManager:
content_parts.append(content)
total_line_count += line_count
# Process code files at the end if configured
if not abort_due_to_max_lines:
code_file_parts, processed_file_paths = self._process_code_files()
code_files_line_count = sum(
part.count("\n") + 1 for part in code_file_parts
)
# Check if adding code files would exceed the maximum line count
max_lines = self.config.get("llms_txt_full_max_size")
if (
max_lines is not None
and total_line_count + code_files_line_count > max_lines
):
logger.warning(
f"sphinx-llms-txt: Adding code files would exceed max line limit "
f"({max_lines}). Current: {total_line_count}, "
f"Code files: {code_files_line_count}. Skipping code files."
)
else:
# Add source code files section if there are any code files
if code_file_parts:
section_header = self._create_code_files_section_header(
processed_file_paths
)
content_parts.append(section_header)
content_parts.extend(code_file_parts)
# Add line count for the section header too
total_line_count += (
code_files_line_count + section_header.count("\n") + 1
)
# Check if line limit was exceeded before creating the file
max_lines = self.config.get("llms_txt_full_max_size")
if abort_due_to_max_lines or (
@@ -370,3 +507,309 @@ class LLMSFullManager:
return source_suffix
else:
return [source_suffix] # String format
def _process_code_files(self) -> Tuple[List[str], List[Path]]:
"""Process code files specified in llms_txt_code_files configuration.
Supports include/exclude patterns with +:/- : prefixes:
- '+:pattern' = include files matching pattern
- '-:pattern' = exclude files matching pattern
- 'pattern' (no prefix) = ignored (no special handling)
Returns:
Tuple of (formatted code block strings, list of processed file paths)
"""
code_file_patterns = self.config.get("llms_txt_code_files", [])
if not code_file_patterns:
return [], []
# Parse patterns into include and exclude lists
include_patterns = []
exclude_patterns = []
for pattern in code_file_patterns:
if pattern.startswith("-:"):
exclude_patterns.append(pattern[2:]) # Remove the '-:' prefix
elif pattern.startswith("+:"):
include_patterns.append(pattern[2:]) # Remove the '+:' prefix
else:
# No prefix = log warning about ignored pattern
logger.warning(
f"sphinx-llms-txt: Code file pattern '{pattern}' ignored."
f"Use '+:{pattern}' to include or '-:{pattern}' to exclude."
)
# If no include patterns specified, nothing to process
if not include_patterns:
return [], []
code_parts = []
processed_files = set()
all_matching_files = set()
# First, collect all files matching include patterns
for pattern in include_patterns:
# Resolve pattern relative to source directory
if self.srcdir:
pattern_path = Path(self.srcdir) / pattern
else:
pattern_path = Path(pattern)
# Use glob to find matching files
matching_files = glob.glob(str(pattern_path), recursive=True)
for file_path_str in matching_files:
file_path = Path(file_path_str)
if file_path.is_file(): # Only add files, not directories
all_matching_files.add(file_path.resolve())
# Filter out files matching exclude patterns
filtered_files = set()
for file_path in all_matching_files:
should_exclude = False
for exclude_pattern in exclude_patterns:
# Resolve exclude pattern relative to source directory
if self.srcdir:
exclude_pattern_path = Path(self.srcdir) / exclude_pattern
else:
exclude_pattern_path = Path(exclude_pattern)
# Check if this file matches the exclude pattern
exclude_matches = glob.glob(str(exclude_pattern_path), recursive=True)
if str(file_path) in exclude_matches:
should_exclude = True
logger.debug(
f"sphinx-llms-txt: Excluding code file: {file_path} "
f"(matched pattern: {exclude_pattern})"
)
break
if not should_exclude:
filtered_files.add(file_path)
# Sort files for consistent ordering
sorted_files = sorted(filtered_files)
for file_path in sorted_files:
# Skip if already processed (shouldn't happen with set, but safety check)
if file_path in processed_files:
continue
try:
# Read the file content
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
content = f.read()
# Get language identifier
language = _get_language_from_extension(file_path)
# Get relative path from source directory for title
if self.srcdir:
try:
title = file_path.relative_to(Path(self.srcdir))
# Strip base path if configured,
# or auto-detect from git root
base_path = self.config.get("llms_txt_code_base_path")
if base_path is None:
# Auto-detect: try to make path relative to git root
git_root = _get_git_root(Path(self.srcdir))
if git_root:
try:
# Get srcdir relative to git root
srcdir_relative = Path(self.srcdir).relative_to(
git_root
)
# Calculate relative path from srcdir to
# git root
if srcdir_relative != Path("."):
# Count directory levels to go up
up_levels = len(srcdir_relative.parts)
base_path = "../" * up_levels
else:
base_path = None
except ValueError:
base_path = None
if base_path:
title_str = str(title)
if title_str.startswith(base_path):
title = Path(title_str[len(base_path) :])
except ValueError:
# File is not relative to srcdir, use filename
title = file_path.name
else:
title = file_path.name
# Format as code block with equals underline
title_str = str(title)
equals_line = "=" * len(title_str)
# Indent the content for reStructuredText code-block directive
indented_content = "\n".join(
f" {line}" if line.strip() else ""
for line in content.splitlines()
)
code_block = f"""
{title_str}
{equals_line}
.. code-block:: {language}
{indented_content}"""
code_parts.append(code_block)
processed_files.add(file_path)
logger.debug(f"sphinx-llms-txt: Added code file: {title}")
except Exception as e:
logger.warning(
f"sphinx-llms-txt: Error reading code file {file_path}: {e}"
)
continue
return code_parts, sorted(processed_files)
def _create_code_files_section_header(self, file_paths: List[Path] = None) -> str:
"""Create the section header for source code files.
Args:
file_paths: List of file paths that were added to generate tree view
Returns:
String containing the section header with title, underlines, description,
and file tree
"""
section_title = "Source Code Files"
star_line = "*" * len(section_title)
description = "This section contains source code files from the project repository. These files are included to provide implementation context and technical details that complement the documentation above." # noqa: E501
header = f"""
{star_line}
{section_title}
{star_line}
{description}"""
# Add file tree if file paths are provided
if file_paths:
tree_display = self._generate_file_tree(file_paths)
header += f"""
**Files included:**
.. code-block:: text
{tree_display}"""
return header
def _generate_file_tree(self, file_paths: List[Path]) -> str:
"""Generate a tree-like representation of file paths.
Args:
file_paths: List of file paths to display in tree format
Returns:
String containing indented tree representation of the files
"""
if not file_paths:
return ""
# Convert to relative paths if possible and create tree structure
tree_data = {}
for file_path in sorted(file_paths):
# Get relative path from source directory for display
if self.srcdir:
try:
rel_path = file_path.relative_to(Path(self.srcdir))
# Apply base path stripping logic similar to code processing
base_path = self.config.get("llms_txt_code_base_path")
if base_path is None:
# Auto-detect: try to make path relative to git root
git_root = _get_git_root(Path(self.srcdir))
if git_root:
try:
# Get srcdir relative to git root
srcdir_relative = Path(self.srcdir).relative_to(
git_root
)
# Calculate relative path from srcdir to git root
if srcdir_relative != Path("."):
# Count directory levels to go up
up_levels = len(srcdir_relative.parts)
base_path = "../" * up_levels
else:
base_path = None
except ValueError:
base_path = None
if base_path:
rel_path_str = str(rel_path)
if rel_path_str.startswith(base_path):
rel_path = Path(rel_path_str[len(base_path) :])
except ValueError:
# File is not relative to srcdir, use filename
rel_path = Path(file_path.name)
else:
rel_path = Path(file_path.name)
# Build nested dictionary structure
parts = rel_path.parts
current = tree_data
for part in parts[:-1]: # All but the last part (directories)
if part not in current:
current[part] = {}
current = current[part]
# Add the file (last part)
if parts:
current[parts[-1]] = None # None indicates it's a file
# Convert tree structure to string representation
lines = []
self._format_tree_node(tree_data, lines, "", True)
# Indent each line for reStructuredText code block
indented_lines = [f" {line}" for line in lines]
return "\n".join(indented_lines)
def _format_tree_node(
self, node: dict, lines: List[str], prefix: str, is_root: bool
):
"""Recursively format tree nodes into lines with proper tree characters.
Args:
node: Dictionary representing the tree structure
lines: List to append formatted lines to
prefix: Current prefix for indentation and tree characters
is_root: Whether this is the root level (no tree characters)
"""
if not node:
return
items = sorted(node.items())
for i, (name, subtree) in enumerate(items):
is_last = i == len(items) - 1
if is_root:
# Root level - no tree characters
current_prefix = ""
next_prefix = ""
else:
# Use tree characters
current_prefix = prefix + ("└── " if is_last else "├── ")
next_prefix = prefix + (" " if is_last else "│ ")
lines.append(current_prefix + name)
# Recursively handle subdirectories
if subtree is not None: # It's a directory
self._format_tree_node(subtree, lines, next_prefix, False)