Refactor LLMSFullManager with clearer class structure (#11)

sphinx_llms_txt/collector.py

@@ -0,0 +1,130 @@
+"""
+Document collector module for sphinx-llms-txt.
+"""
+
+import fnmatch
+from typing import Any, Dict, List
+
+from sphinx.environment import BuildEnvironment
+from sphinx.util import logging
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentCollector:
+    """Collects and orders documentation sources based on toctree structure."""
+
+    def __init__(self):
+        self.page_titles: Dict[str, str] = {}
+        self.master_doc: str = None
+        self.env: BuildEnvironment = None
+        self.config: Dict[str, Any] = {}
+
+    def set_master_doc(self, master_doc: str):
+        """Set the master document name."""
+        self.master_doc = master_doc
+
+    def set_env(self, env: BuildEnvironment):
+        """Set the Sphinx environment."""
+        self.env = env
+
+    def update_page_title(self, docname: str, title: str):
+        """Update the title for a page."""
+        if title:
+            self.page_titles[docname] = title
+
+    def set_config(self, config: Dict[str, Any]):
+        """Set configuration options."""
+        self.config = config
+
+    def get_page_order(self) -> List[str]:
+        """Get the correct page order from the toctree structure."""
+        if not self.env or not self.master_doc:
+            return []
+
+        page_order = []
+        visited = set()
+
+        def collect_from_toctree(docname: str):
+            """Recursively collect documents from toctree."""
+            if docname in visited:
+                return
+
+            visited.add(docname)
+
+            # Add the current document
+            if docname not in page_order:
+                page_order.append(docname)
+
+            # Check for toctree entries in this document
+            try:
+                # Look for toctree_includes which contains the direct children
+                if (
+                    hasattr(self.env, "toctree_includes")
+                    and docname in self.env.toctree_includes
+                ):
+                    for child_docname in self.env.toctree_includes[docname]:
+                        collect_from_toctree(child_docname)
+                else:
+                    # Fallback: try to resolve and parse the toctree
+                    toctree = self.env.get_and_resolve_toctree(docname, None)
+                    if toctree:
+                        from docutils import nodes
+
+                        for node in list(toctree.findall(nodes.reference)):
+                            if "refuri" in node.attributes:
+                                refuri = node.attributes["refuri"]
+                                if refuri and refuri.endswith(".html"):
+                                    child_docname = refuri[:-5]  # Remove .html
+                                    if (
+                                        child_docname != docname
+                                    ):  # Avoid circular references
+                                        collect_from_toctree(child_docname)
+            except Exception as e:
+                logger.debug(f"Could not get toctree for {docname}: {e}")
+
+        # Start from the master document
+        collect_from_toctree(self.master_doc)
+
+        # Add any remaining documents not in the toctree (sorted)
+        if hasattr(self.env, "all_docs"):
+            remaining = sorted(
+                [doc for doc in self.env.all_docs.keys() if doc not in page_order]
+            )
+            page_order.extend(remaining)
+
+        return page_order
+
+    def filter_excluded_pages(self, page_order: List[str]) -> List[str]:
+        """Filter out excluded pages from the page order."""
+        exclude_patterns = self.config.get("llms_txt_exclude")
+        if exclude_patterns:
+            return [
+                page
+                for page in page_order
+                if not any(
+                    self._match_exclude_pattern(page, pattern)
+                    for pattern in exclude_patterns
+                )
+            ]
+        return page_order
+
+    def _match_exclude_pattern(self, docname: str, pattern: str) -> bool:
+        """Check if a document name matches an exclude pattern.
+
+        Args:
+            docname: The document name to check
+            pattern: The pattern to match against
+
+        Returns:
+            True if the document should be excluded, False otherwise
+        """
+        # Exact match
+        if docname == pattern:
+            return True
+
+        # Glob-style pattern matching
+        if fnmatch.fnmatch(docname, pattern):
+            return True
+
+        return False