From b56d93d265cc29ca6a2799b115f35f267e995a5f Mon Sep 17 00:00:00 2001 From: Jared Dillard Date: Sun, 22 Jun 2025 19:28:54 -0700 Subject: [PATCH] Support source file suffix detection (#21) --- CHANGELOG.rst | 6 + sphinx_llms_txt/__init__.py | 2 +- sphinx_llms_txt/collector.py | 104 +++++++-- sphinx_llms_txt/manager.py | 120 +++++++---- sphinx_llms_txt/writer.py | 15 +- tests/test_llms_txt.py | 393 +++++++++++++++++++++++++++++++++++ 6 files changed, 585 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47d72a3..cc5f92f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +0.2.4 +----- + +- Support source file suffix detection + `#21 `_ + 0.2.3 ----- diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index cd22884..160ed90 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -12,7 +12,7 @@ from .manager import LLMSFullManager from .processor import DocumentProcessor from .writer import FileWriter -__version__ = "0.2.3" +__version__ = "0.2.4" # Export classes needed by tests __all__ = [ diff --git a/sphinx_llms_txt/collector.py b/sphinx_llms_txt/collector.py index 9636414..144938f 100644 --- a/sphinx_llms_txt/collector.py +++ b/sphinx_llms_txt/collector.py @@ -3,7 +3,7 @@ Document collector module for sphinx-llms-txt. """ import fnmatch -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple from sphinx.environment import BuildEnvironment from sphinx.util import logging @@ -19,6 +19,7 @@ class DocumentCollector: self.master_doc: str = None self.env: BuildEnvironment = None self.config: Dict[str, Any] = {} + self.app = None def set_master_doc(self, master_doc: str): """Set the master document name.""" @@ -37,8 +38,73 @@ class DocumentCollector: """Set configuration options.""" self.config = config - def get_page_order(self) -> List[str]: - """Get the correct page order from the toctree structure.""" + def set_app(self, app): + """Set the Sphinx application reference.""" + self.app = app + + def _get_source_suffixes(self): + """Get all valid source file suffixes from Sphinx configuration. + + Returns: + list: List of source file suffixes (e.g., ['.rst', '.md', '.txt']) + """ + if not self.app: + return [".rst"] # Default fallback + + source_suffix = self.app.config.source_suffix + + if isinstance(source_suffix, dict): + return list(source_suffix.keys()) + elif isinstance(source_suffix, list): + return source_suffix + else: + return [source_suffix] # String format + + def _get_docname_suffix(self, docname: str, sources_dir) -> str: + """ + Determine the source suffix for a given docname by checking which + file exists. + + Args: + docname: The document name to check + sources_dir: Path to the _sources directory + + Returns: + The source suffix if found, or None if no matching file exists + """ + if not sources_dir or not sources_dir.exists(): + return None + + # Get the source link suffix from Sphinx config + source_link_suffix = "" + if self.app and hasattr(self.app.config, "html_sourcelink_suffix"): + source_link_suffix = self.app.config.html_sourcelink_suffix + # Handle empty string case specially + if source_link_suffix == "": + source_link_suffix = "" # Keep it empty + elif not source_link_suffix.startswith("."): + source_link_suffix = "." + source_link_suffix + + # Get the source file suffixes from Sphinx config + source_suffixes = self._get_source_suffixes() + + # Try to find the source file with any of the valid source suffixes + for src_suffix in source_suffixes: + candidate_file = sources_dir / f"{docname}{src_suffix}{source_link_suffix}" + if candidate_file.exists(): + return src_suffix + + return None + + def get_page_order(self, sources_dir=None) -> List[Tuple[str, str]]: + """Get the correct page order from the toctree structure. + + Args: + sources_dir: Optional path to _sources directory for suffix detection + + Returns: + List of tuples (docname, source_suffix) in toctree order + """ if not self.env or not self.master_doc: return [] @@ -52,9 +118,12 @@ class DocumentCollector: visited.add(docname) - # Add the current document - if docname not in page_order: - page_order.append(docname) + # Add the current document with its suffix + if docname not in [doc for doc, _ in page_order]: + suffix = None + if sources_dir: + suffix = self._get_docname_suffix(docname, sources_dir) + page_order.append((docname, suffix)) # Check for toctree entries in this document try: @@ -101,22 +170,33 @@ class DocumentCollector: # Add any remaining documents not in the toctree (sorted) if hasattr(self.env, "all_docs"): + processed_docnames = {doc for doc, _ in page_order} remaining = sorted( - [doc for doc in self.env.all_docs.keys() if doc not in page_order] + [ + doc + for doc in self.env.all_docs.keys() + if doc not in processed_docnames + ] ) - page_order.extend(remaining) + for docname in remaining: + suffix = None + if sources_dir: + suffix = self._get_docname_suffix(docname, sources_dir) + page_order.append((docname, suffix)) return page_order - def filter_excluded_pages(self, page_order: List[str]) -> List[str]: + def filter_excluded_pages( + self, page_order: List[Tuple[str, str]] + ) -> List[Tuple[str, str]]: """Filter out excluded pages from the page order.""" exclude_patterns = self.config.get("llms_txt_exclude") if exclude_patterns: return [ - page - for page in page_order + (docname, suffix) + for docname, suffix in page_order if not any( - self._match_exclude_pattern(page, pattern) + self._match_exclude_pattern(docname, pattern) for pattern in exclude_patterns ) ] diff --git a/sphinx_llms_txt/manager.py b/sphinx_llms_txt/manager.py index f4f3d8a..47fb766 100644 --- a/sphinx_llms_txt/manager.py +++ b/sphinx_llms_txt/manager.py @@ -56,6 +56,7 @@ class LLMSFullManager: def set_app(self, app: Sphinx): """Set the Sphinx application reference.""" self.app = app + self.collector.set_app(app) if self.writer: self.writer.app = app @@ -69,23 +70,7 @@ class LLMSFullManager: self.processor = DocumentProcessor(self.config, srcdir) self.writer = FileWriter(self.config, outdir, self.app) - # Get the correct page order - page_order = self.collector.get_page_order() - - if not page_order: - logger.warning( - "Could not determine page order, skipping llms-full creation" - ) - return - - # Apply exclusion filter if configured - page_order = self.collector.filter_excluded_pages(page_order) - - # Determine output file name and location - output_filename = self.config.get("llms_txt_full_filename") - output_path = Path(outdir) / output_filename - - # Find sources directory + # Find sources directory first so we can pass it to get_page_order sources_dir = None possible_sources = [ Path(outdir) / "_sources", @@ -104,6 +89,22 @@ class LLMSFullManager: ) return + # Get the correct page order with source suffixes + page_order = self.collector.get_page_order(sources_dir) + + if not page_order: + logger.warning( + "Could not determine page order, skipping llms-full creation" + ) + return + + # Apply exclusion filter if configured + page_order = self.collector.filter_excluded_pages(page_order) + + # Determine output file name and location + output_filename = self.config.get("llms_txt_full_filename") + output_path = Path(outdir) / output_filename + # Log discovered files and page order logger.debug(f"sphinx-llms-txt: Page order (after exclusion): {page_order}") @@ -115,8 +116,19 @@ class LLMSFullManager: # Create a mapping from docnames to source files docname_to_file = {} - # Process each docname in the page order - for docname in page_order: + # Get the source link suffix from Sphinx config + source_link_suffix = ( + self.app.config.html_sourcelink_suffix if self.app else ".txt" + ) + + # Handle empty string case specially + if source_link_suffix == "": + source_link_suffix = "" # Keep it empty + elif not source_link_suffix.startswith("."): + source_link_suffix = "." + source_link_suffix + + # Process each (docname, suffix) in the page order + for docname, src_suffix in page_order: # Skip excluded pages if exclude_patterns and any( self.collector._match_exclude_pattern(docname, pattern) @@ -124,15 +136,19 @@ class LLMSFullManager: ): continue - # Construct expected source file path directly from docname - source_file = sources_dir / f"{docname}.rst.txt" - - if source_file.exists(): - docname_to_file[docname] = source_file + # Build the source file path directly using the known suffix + if src_suffix: + source_file = sources_dir / f"{docname}{src_suffix}{source_link_suffix}" + if source_file.exists(): + docname_to_file[docname] = source_file + else: + logger.warning( + f"sphinx-llms-txt: Source file not found for: {docname}." + f"Expected: {docname}{src_suffix}{source_link_suffix}" + ) else: logger.warning( - f"sphinx-llm-txt: Source file not found for: {docname}. Expected" - f" at {source_file}" + f"sphinx-llms-txt: No source suffix determined for: {docname}" ) # Generate content @@ -144,7 +160,7 @@ class LLMSFullManager: max_lines = self.config.get("llms_txt_full_max_size") abort_due_to_max_lines = False - for docname in page_order: + for docname, _ in page_order: if docname in docname_to_file: file_path = docname_to_file[docname] content, line_count = self._read_source_file(file_path, docname) @@ -179,14 +195,19 @@ class LLMSFullManager: total_line_count += line_count else: logger.warning( - f"sphinx-llm-txt: Source file not found for: {docname}. Check that" - f" the file exists at _sources/{docname}.rst.txt" + f"sphinx-llms-txt: Source file not found for: {docname}. Check that" + f" file exists at _sources/{docname}[suffix]{source_link_suffix}" ) # Add any remaining files (in alphabetical order) that aren't in the page order if not abort_due_to_max_lines: - # Get all .rst.txt files in the _sources directory - all_source_files = list(sources_dir.glob("**/*.rst.txt")) + # Get all source files in the _sources directory using configured suffixes + source_suffixes = self._get_source_suffixes() + all_source_files = [] + for src_suffix in source_suffixes: + glob_pattern = f"**/*{src_suffix}{source_link_suffix}" + all_source_files.extend(sources_dir.glob(glob_pattern)) + processed_paths = set(file.resolve() for file in docname_to_file.values()) # Find files that haven't been processed yet @@ -204,11 +225,18 @@ class LLMSFullManager: ) for file_path in remaining_source_files: - # Extract docname from path by removing the .rst.txt extension + # Extract docname from path by removing the source and link suffixes rel_path = str(file_path.relative_to(sources_dir)) - if rel_path.endswith(".rst.txt"): - docname = rel_path[:-8] # Remove .rst.txt extension - else: + docname = None + + # Try each source suffix to find which one this file uses + for src_suffix in source_suffixes: + combined_suffix = f"{src_suffix}{source_link_suffix}" + if rel_path.endswith(combined_suffix): + docname = rel_path[: -len(combined_suffix)] # Remove suffix + break + + if docname is None: continue # Skip excluded docnames @@ -237,7 +265,7 @@ class LLMSFullManager: max_lines is not None and total_line_count > max_lines ): logger.warning( - f"sphinx-llm-txt: Max line limit ({max_lines}) exceeded:" + f"sphinx-llms-txt: Max line limit ({max_lines}) exceeded:" f" {total_line_count} > {max_lines}. " f"Not creating llms-full.txt file." ) @@ -304,5 +332,23 @@ class LLMSFullManager: return content_str, line_count + 1 except Exception as e: - logger.error(f"sphinx-llm-txt: Error reading source file {file_path}: {e}") + logger.error(f"sphinx-llms-txt: Error reading source file {file_path}: {e}") return "", 0 + + def _get_source_suffixes(self): + """Get all valid source file suffixes from Sphinx configuration. + + Returns: + list: List of source file suffixes (e.g., ['.rst', '.md', '.txt']) + """ + if not self.app: + return [".rst"] # Default fallback + + source_suffix = self.app.config.source_suffix + + if isinstance(source_suffix, dict): + return list(source_suffix.keys()) + elif isinstance(source_suffix, list): + return source_suffix + else: + return [source_suffix] # String format diff --git a/sphinx_llms_txt/writer.py b/sphinx_llms_txt/writer.py index 8d45eea..1828bc0 100644 --- a/sphinx_llms_txt/writer.py +++ b/sphinx_llms_txt/writer.py @@ -3,7 +3,7 @@ File writer module for sphinx-llms-txt. """ from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple, Union from sphinx.application import Sphinx from sphinx.util import logging @@ -42,19 +42,19 @@ class FileWriter: ) return True except Exception as e: - logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}") + logger.error(f"sphinx-llms-txt: Error writing combined sources file: {e}") return False def write_verbose_info_to_file( self, - page_order: List[str], + page_order: Union[List[str], List[Tuple[str, str]]], page_titles: Dict[str, str], total_line_count: int = 0, ) -> bool: """Write summary information to the llms.txt file. Args: - page_order: Ordered list of document names + page_order: Ordered list of document names or (docname, suffix) tuples page_titles: Dictionary mapping docnames to titles total_line_count: Total number of lines in the combined content @@ -100,7 +100,12 @@ class FileWriter: if not base_url.endswith("/"): base_url += "/" - for docname in page_order: + for item in page_order: + # Handle both old format (str) and new format (tuple) + if isinstance(item, tuple): + docname, _ = item + else: + docname = item title = page_titles.get(docname, docname) f.write(f"- [{title}]({base_url}{docname}.html)\n") diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index ddd07a1..0c9c9b9 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -334,3 +334,396 @@ def test_write_verbose_info_with_baseurl(tmp_path): assert "- [Home Page](https://example.org/index.html)" in content assert "- [About Us](https://example.org/about.html)" in content + + +def test_get_source_suffixes_with_dict(): + """Test _get_source_suffixes method with dict source_suffix.""" + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with dict source_suffix + class MockApp: + class Config: + source_suffix = {".rst": None, ".md": None, ".txt": None} + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + + suffixes = manager._get_source_suffixes() + assert set(suffixes) == {".rst", ".md", ".txt"} + + +def test_get_source_suffixes_with_list(): + """Test _get_source_suffixes method with list source_suffix.""" + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with list source_suffix + class MockApp: + class Config: + source_suffix = [".rst", ".md"] + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + + suffixes = manager._get_source_suffixes() + assert suffixes == [".rst", ".md"] + + +def test_get_source_suffixes_with_string(): + """Test _get_source_suffixes method with string source_suffix.""" + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with string source_suffix + class MockApp: + class Config: + source_suffix = ".rst" + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + + suffixes = manager._get_source_suffixes() + assert suffixes == [".rst"] + + +def test_get_source_suffixes_no_app(): + """Test _get_source_suffixes method with no app set.""" + from sphinx_llms_txt.manager import LLMSFullManager + + manager = LLMSFullManager() + + suffixes = manager._get_source_suffixes() + assert suffixes == [".rst"] # Default fallback + + +def test_html_sourcelink_suffix_default(): + """Test html_sourcelink_suffix defaults to .txt when no app is set.""" + import tempfile + + from sphinx_llms_txt.manager import LLMSFullManager + + manager = LLMSFullManager() + manager.set_config( + { + "llms_txt_full_filename": "test.txt", + "llms_txt_exclude": [], + "llms_txt_directives": [], + } + ) + + # Create a temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + outdir = f"{tmpdir}/build" + srcdir = f"{tmpdir}/source" + sources_dir = f"{outdir}/_sources" + + # Create directories + import os + + os.makedirs(sources_dir, exist_ok=True) + os.makedirs(srcdir, exist_ok=True) + + # Create a test source file with default .txt suffix + test_file = f"{sources_dir}/index.rst.txt" + with open(test_file, "w") as f: + f.write("Test content") + + # Mock env with minimal required attributes + class MockEnv: + all_docs = {"index": None} + titles = { + "index": type("TitleNode", (), {"astext": lambda: "Test Title"})() + } + toctree_includes = {} + + manager.set_env(MockEnv()) + manager.set_master_doc("index") + + # Test that it uses .txt as the default suffix + manager.combine_sources(outdir, srcdir) + + # Verify the file was found and processed (check if output file exists) + output_file = f"{outdir}/test.txt" + assert os.path.exists(output_file) + + +def test_html_sourcelink_suffix_custom(): + """Test html_sourcelink_suffix uses custom value from Sphinx config.""" + import tempfile + + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with custom html_sourcelink_suffix + class MockApp: + class Config: + html_sourcelink_suffix = "source" + source_suffix = ".rst" + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + manager.set_config( + { + "llms_txt_full_filename": "test.txt", + "llms_txt_exclude": [], + "llms_txt_directives": [], + } + ) + + # Create a temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + outdir = f"{tmpdir}/build" + srcdir = f"{tmpdir}/source" + sources_dir = f"{outdir}/_sources" + + # Create directories + import os + + os.makedirs(sources_dir, exist_ok=True) + os.makedirs(srcdir, exist_ok=True) + + # Create a test source file with custom .source suffix + test_file = f"{sources_dir}/index.rst.source" + with open(test_file, "w") as f: + f.write("Test content") + + # Mock env with minimal required attributes + class MockEnv: + all_docs = {"index": None} + titles = { + "index": type("TitleNode", (), {"astext": lambda: "Test Title"})() + } + toctree_includes = {} + + manager.set_env(MockEnv()) + manager.set_master_doc("index") + + # Test that it uses .source as the custom suffix + manager.combine_sources(outdir, srcdir) + + # Verify the file was found and processed + output_file = f"{outdir}/test.txt" + assert os.path.exists(output_file) + + +def test_html_sourcelink_suffix_with_dot(): + """Test html_sourcelink_suffix adds dot if missing.""" + import tempfile + + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with html_sourcelink_suffix without leading dot + class MockApp: + class Config: + html_sourcelink_suffix = "src" # No leading dot + source_suffix = ".rst" + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + manager.set_config( + { + "llms_txt_full_filename": "test.txt", + "llms_txt_exclude": [], + "llms_txt_directives": [], + } + ) + + # Create a temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + outdir = f"{tmpdir}/build" + srcdir = f"{tmpdir}/source" + sources_dir = f"{outdir}/_sources" + + # Create directories + import os + + os.makedirs(sources_dir, exist_ok=True) + os.makedirs(srcdir, exist_ok=True) + + # Create a test source file with .src suffix (dot should be added automatically) + test_file = f"{sources_dir}/index.rst.src" + with open(test_file, "w") as f: + f.write("Test content") + + # Mock env with minimal required attributes + class MockEnv: + all_docs = {"index": None} + titles = { + "index": type("TitleNode", (), {"astext": lambda: "Test Title"})() + } + toctree_includes = {} + + manager.set_env(MockEnv()) + manager.set_master_doc("index") + + # Test that it adds the dot and finds the file + manager.combine_sources(outdir, srcdir) + + # Verify the file was found and processed + output_file = f"{outdir}/test.txt" + assert os.path.exists(output_file) + + +def test_mixed_source_file_formats(): + """Test handling of mixed source file formats (.rst, .md, .txt).""" + import tempfile + + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with multiple source suffixes + class MockApp: + class Config: + html_sourcelink_suffix = ".txt" + source_suffix = {".rst": None, ".md": None, ".txt": None} + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + manager.set_config( + { + "llms_txt_full_filename": "test.txt", + "llms_txt_exclude": [], + "llms_txt_directives": [], + } + ) + + # Create a temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + outdir = f"{tmpdir}/build" + srcdir = f"{tmpdir}/source" + sources_dir = f"{outdir}/_sources" + + # Create directories + import os + + os.makedirs(sources_dir, exist_ok=True) + os.makedirs(srcdir, exist_ok=True) + + # Create test source files with different formats + files_to_create = [ + f"{sources_dir}/page1.rst.txt", + f"{sources_dir}/page2.md.txt", + f"{sources_dir}/page3.txt.txt", + ] + + for test_file in files_to_create: + with open(test_file, "w") as f: + f.write(f"Content for {os.path.basename(test_file)}") + + # Mock env with all documents + class MockEnv: + all_docs = {"page1": None, "page2": None, "page3": None} + titles = { + "page1": type("TitleNode", (), {"astext": lambda: "Page 1"})(), + "page2": type("TitleNode", (), {"astext": lambda: "Page 2"})(), + "page3": type("TitleNode", (), {"astext": lambda: "Page 3"})(), + } + toctree_includes = {} + + manager.set_env(MockEnv()) + manager.set_master_doc("page1") + + # Test that all file formats are found and processed + manager.combine_sources(outdir, srcdir) + + # Verify the output file was created and contains content from all formats + output_file = f"{outdir}/test.txt" + assert os.path.exists(output_file) + + with open(output_file, "r") as f: + content = f.read() + + # Should contain content from all three files + assert "Content for page1.rst.txt" in content + assert "Content for page2.md.txt" in content + assert "Content for page3.txt.txt" in content + + +def test_source_suffix_detection_priority(): + """Test source suffix detection tries formats in correct order for docnames.""" + import tempfile + + from sphinx_llms_txt.manager import LLMSFullManager + + # Mock Sphinx app with ordered source suffixes + class MockApp: + class Config: + html_sourcelink_suffix = ".txt" + source_suffix = [".rst", ".md"] # rst has priority over md + + config = Config() + + manager = LLMSFullManager() + manager.set_app(MockApp()) + manager.set_config( + { + "llms_txt_full_filename": "test.txt", + "llms_txt_exclude": [], + "llms_txt_directives": [], + } + ) + + # Create a temporary directory structure + with tempfile.TemporaryDirectory() as tmpdir: + outdir = f"{tmpdir}/build" + srcdir = f"{tmpdir}/source" + sources_dir = f"{outdir}/_sources" + + # Create directories + import os + + os.makedirs(sources_dir, exist_ok=True) + os.makedirs(srcdir, exist_ok=True) + + # Create both .rst and .md versions of the same document + # Only create files for the specific docname "index" + rst_file = f"{sources_dir}/index.rst.txt" + md_file = f"{sources_dir}/index.md.txt" + + with open(rst_file, "w") as f: + f.write("RST content for index") + + with open(md_file, "w") as f: + f.write("Markdown content for index") + + # Mock env with only the index document + class MockEnv: + all_docs = {"index": None} + titles = { + "index": type("TitleNode", (), {"astext": lambda: "Index Page"})() + } + toctree_includes = {"index": []} + + manager.set_env(MockEnv()) + manager.set_master_doc("index") + + # Test the priority behavior + manager.combine_sources(outdir, srcdir) + + # Check that output file was created + output_file = f"{outdir}/test.txt" + assert os.path.exists(output_file) + + with open(output_file, "r") as f: + content = f.read() + + # The system should prefer RST over MD for the "index" docname + # But since both files exist and the second phase adds remaining files, + # both will be included. The test verifies that RST appears first + # (indicating it was found first in the priority order) + assert "RST content for index" in content + + # Find positions to verify order + rst_pos = content.find("RST content for index") + md_pos = content.find("Markdown content for index") + + # RST should come before MD (due to priority in toctree processing) + assert rst_pos < md_pos, "RST content should appear before MD content"