add soft transfer

CHANGELOG.rst

@@ -1,16 +1,6 @@
 Changelog
 =========
 
-0.2.3
------
-
-- Remove ``get_and_resolve_toctree`` method
-  `#19 <https://github.com/jdillard/sphinx-llms-txt/pull/19>`_
-- Simplify ``_sources`` lookup
-  `#18 <https://github.com/jdillard/sphinx-llms-txt/pull/18>`_
-- Add sphinx docs
-  `#16 <https://github.com/jdillard/sphinx-llms-txt/pull/16>`_
-
 0.2.2
 -----
 

README.md

@@ -1,6 +1,6 @@
 # Sphinx llms.txt generator
 
-A Sphinx extension that generates a summary `llms.txt` file and a single combined documentation `llms-full.txt` file.
+A Sphinx extension that generates a summary `llms.txt` file, written in Markdown, and a single combined documentation `llms-full.txt` file, written in reStructuredText.
 
 [![PyPI version](https://img.shields.io/pypi/v/sphinx-llms-txt.svg)](https://pypi.python.org/pypi/sphinx-llms-txt)
 [![Downloads](https://static.pepy.tech/badge/sphinx-llms-txt/month)](https://pepy.tech/project/sphinx-llms-txt)

docs/source/advanced-configuration.rst

@@ -146,7 +146,7 @@ Integration Examples
 Complete Configuration Example
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Here's a complete example showing multiple :ref:`configuration-values`:
+Here's a complete example showing multiple configuration options:
 
 .. code-block:: python
 

sphinx_llms_txt/__init__.py

@@ -12,7 +12,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.2.3"
+__version__ = "0.2.2"
 
 # Export classes needed by tests
 __all__ = [

sphinx_llms_txt/collector.py

@@ -65,33 +65,20 @@ class DocumentCollector:
                 ):
                     for child_docname in self.env.toctree_includes[docname]:
                         collect_from_toctree(child_docname)
-                # Try to use dependencies to find related documents
-                elif (
-                    hasattr(self.env, "dependencies")
-                    and docname in self.env.dependencies
-                ):
-                    # Extract the dependent documents from the dependencies dict
-                    for child_docname in self.env.dependencies[docname]:
-                        # Only add documents actually in the document set
-                        if (
-                            hasattr(self.env, "all_docs")
-                            and child_docname in self.env.all_docs
-                        ):
-                            collect_from_toctree(child_docname)
-                # Fallback to titles or other available references
-                elif hasattr(self.env, "titles") and hasattr(self.env, "all_docs"):
-                    # Get all document names
-                    all_docnames = list(self.env.all_docs.keys())
+                else:
+                    # Fallback: try to resolve and parse the toctree
+                    toctree = self.env.get_and_resolve_toctree(docname, None)
+                    if toctree:
+                        from docutils import nodes
 
-                    # Look for documents that might be related (have similar paths)
-                    current_prefix = "/".join(docname.split("/")[:-1])
-                    if current_prefix:
-                        for child_docname in all_docnames:
-                            # Documents in the same directory might be related
+                        for node in list(toctree.findall(nodes.reference)):
+                            if "refuri" in node.attributes:
+                                refuri = node.attributes["refuri"]
+                                if refuri and refuri.endswith(".html"):
+                                    child_docname = refuri[:-5]  # Remove .html
                                     if (
-                                child_docname.startswith(current_prefix)
-                                and child_docname != docname
-                            ):
+                                        child_docname != docname
+                                    ):  # Avoid circular references
                                         collect_from_toctree(child_docname)
             except Exception as e:
                 logger.debug(f"Could not get toctree for {docname}: {e}")

sphinx_llms_txt/manager.py

@@ -104,7 +104,14 @@ class LLMSFullManager:
             )
             return
 
+        # Collect all available source files
+        txt_files = {}
+        for f in sources_dir.glob("**/*.txt"):
+            logger.debug(f"sphinx-llms-txt: Found source file: {f.stem} at {f}")
+            txt_files[f.stem] = f
+
         # Log discovered files and page order
+        logger.debug(f"sphinx-llms-txt: Found {len(txt_files)} source files")
         logger.debug(f"sphinx-llms-txt: Page order (after exclusion): {page_order}")
 
         # Log exclusion patterns
@@ -112,28 +119,33 @@ class LLMSFullManager:
         if exclude_patterns:
             logger.debug(f"sphinx-llms-txt: Exclusion patterns: {exclude_patterns}")
 
-        # Create a mapping from docnames to source files
+        # Create a mapping from docnames to actual file names
         docname_to_file = {}
 
-        # Process each docname in the page order
+        # Try exact matches first
         for docname in page_order:
             # Skip excluded pages
-            if exclude_patterns and any(
+            if any(
                 self.collector._match_exclude_pattern(docname, pattern)
                 for pattern in exclude_patterns
             ):
                 continue
 
-            # Construct expected source file path directly from docname
-            source_file = sources_dir / f"{docname}.rst.txt"
-
-            if source_file.exists():
-                docname_to_file[docname] = source_file
+            if docname in txt_files:
+                docname_to_file[docname] = txt_files[docname]
             else:
-                logger.warning(
-                    f"sphinx-llm-txt: Source file not found for: {docname}. Expected"
-                    f" at {source_file}"
-                )
+                # Try with .rst extension
+                if f"{docname}.rst" in txt_files:
+                    docname_to_file[docname] = txt_files[f"{docname}.rst"]
+                # Try with .txt extension
+                elif f"{docname}.txt" in txt_files:
+                    docname_to_file[docname] = txt_files[f"{docname}.txt"]
+                # Try with underscores instead of hyphens
+                elif docname.replace("-", "_") in txt_files:
+                    docname_to_file[docname] = txt_files[docname.replace("-", "_")]
+                # Try with hyphens instead of underscores
+                elif docname.replace("_", "-") in txt_files:
+                    docname_to_file[docname] = txt_files[docname.replace("_", "-")]
 
         # Generate content
         content_parts = []
@@ -178,56 +190,65 @@ class LLMSFullManager:
                     added_files.add(file_path.stem)
                     total_line_count += line_count
             else:
-                logger.warning(
-                    f"sphinx-llm-txt: Source file not found for: {docname}. Check that"
-                    f" the file exists at _sources/{docname}.rst.txt"
-                )
+                logger.warning(f"sphinx-llm-txt: Source file not found for: {docname}")
 
-        # Add any remaining files (in alphabetical order) that aren't in the page order
+        # Add any remaining files (in alphabetical order) if not aborted
         if not abort_due_to_max_lines:
-            # Get all .rst.txt files in the _sources directory
-            all_source_files = list(sources_dir.glob("**/*.rst.txt"))
-            processed_paths = set(file.resolve() for file in docname_to_file.values())
+            # Apply the same exclusion filter to remaining files
+            exclude_patterns = self.config.get("llms_txt_exclude")
 
-            # Find files that haven't been processed yet
-            remaining_source_files = [
-                f for f in all_source_files if f.resolve() not in processed_paths
-            ]
+            # Create a set of files to exclude based on their basename
+            excluded_files = set()
+            for pattern in exclude_patterns:
+                if "*" not in pattern and "?" not in pattern:
+                    # For exact patterns, add variants
+                    excluded_files.add(pattern)
+                    excluded_files.add(f"{pattern}.rst")
+                    excluded_files.add(f"{pattern}.txt")
+                    excluded_files.add(pattern.replace("-", "_"))
+                    excluded_files.add(pattern.replace("_", "-"))
 
-            # Sort the remaining files for consistent ordering
-            remaining_source_files.sort()
-
-            if remaining_source_files:
-                logger.info(
-                    f"Found {len(remaining_source_files)} additional files not in"
-                    f" toctree"
-                )
-
-            for file_path in remaining_source_files:
-                # Extract docname from path by removing the .rst.txt extension
-                rel_path = str(file_path.relative_to(sources_dir))
-                if rel_path.endswith(".rst.txt"):
-                    docname = rel_path[:-8]  # Remove .rst.txt extension
-                else:
-                    continue
-
-                # Skip excluded docnames
-                if exclude_patterns and any(
-                    self.collector._match_exclude_pattern(docname, pattern)
+            # Filter remaining files
+            remaining_files = sorted(
+                [
+                    name
+                    for name in txt_files
+                    if name not in added_files
+                    and name not in excluded_files
+                    and not any(
+                        self.collector._match_exclude_pattern(name, pattern)
                         for pattern in exclude_patterns
-                ):
-                    logger.debug(f"sphinx-llms-txt: Skipping excluded file: {docname}")
-                    continue
-
-                # Read and process the file
-                content, line_count = self._read_source_file(file_path, docname)
+                    )
+                ]
+            )
+            if remaining_files:
+                logger.info(f"Adding remaining files: {remaining_files}")
+            for file_stem in remaining_files:
+                file_path = txt_files[file_stem]
+                content, line_count = self._read_source_file(file_path, file_stem)
 
                 # Check if adding this file would exceed the maximum line count
                 if max_lines is not None and total_line_count + line_count > max_lines:
                     break
 
-                if content:
-                    logger.debug(f"sphinx-llms-txt: Adding remaining file: {docname}")
+                # Double-check that this file should be included
+                should_include = True
+                file_stem = file_path.stem
+                exclude_patterns = self.config.get("llms_txt_exclude")
+
+                if exclude_patterns:
+                    # Check stem against exclusion patterns
+                    if any(
+                        self.collector._match_exclude_pattern(file_stem, pattern)
+                        for pattern in exclude_patterns
+                    ):
+                        logger.debug(
+                            "sphinx-llms-txt: Final exclusion check removed remaining"
+                            f" file: {file_stem}"
+                        )
+                        should_include = False
+
+                if content and should_include:
                     content_parts.append(content)
                     total_line_count += line_count