Add ability to exclude pages (#10)

CHANGELOG.rst

@@ -1,13 +1,18 @@
 Changelog
 =========
 
+0.2.1
+-----
+
+- Add ability to exclude pages with ``llms_txt_exclude``
+
 0.2.0
 -----
 
-- Add `llms_txt_full_max_size` configuration option to limit `llms-full.txt` file size
-- Automatically add content from `include` directives in  `llms-full.txt`
-- Add path resolution for a given set of directives  in `llms-full.txt`
-- Add `llms.txt` file option, with `llms_txt_title` and `llms_txt_summary` config values
+- Add ``llms_txt_full_max_size`` configuration option to limit `llms-full.txt` file size
+- Automatically add content from **include** directives in  **llms-full.txt**
+- Add path resolution for a given set of directives  in **llms-full.txt**
+- Add **llms.txt** file option, with ``llms_txt_title`` and ``llms_txt_summary`` config values
 
 0.1.0
 -----

README.md

@@ -69,6 +69,12 @@ extensions = [
 - **Default**: `None`
 - **Description**: Optional, but recommended, summary description for `llms.txt`.
 
+### `llms_txt_exclude`
+
+- **Type**: list of strings
+- **Default**: `[]`
+- **Description**: A list of pages to ignore  (e.g., "page1", "page_with_*").
+
 ## Features
 
 - Creates `llms.txt` and `llms-full.txt`
@@ -76,6 +82,7 @@ extensions = [
 - Resolves relative paths in directives like `image` and `figure` to use full paths
   - Ability to add list of custom directives with `llms_txt_directives`
   - Optionally, prepend a base URL using Sphinx's `html_baseurl`
+- Ability to exclude pages
 
 ## License
 

sphinx_llms_txt/__init__.py

@@ -11,7 +11,7 @@ from sphinx.application import Sphinx
 from sphinx.environment import BuildEnvironment
 from sphinx.util import logging
 
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 logger = logging.getLogger(__name__)
 
@@ -122,6 +122,18 @@ class LLMSFullManager:
             )
             return
 
+        # Apply exclusion filter if configured
+        exclude_patterns = self.config.get("llms_txt_exclude")
+        if exclude_patterns:
+            page_order = [
+                page
+                for page in page_order
+                if not any(
+                    self._match_exclude_pattern(page, pattern)
+                    for pattern in exclude_patterns
+                )
+            ]
+
         # Determine output file name and location
         output_filename = self.config.get("llms_txt_full_filename")
         output_path = Path(outdir) / output_filename
@@ -148,13 +160,30 @@ class LLMSFullManager:
         # Collect all available source files
         txt_files = {}
         for f in sources_dir.glob("*.txt"):
+            logger.debug(f"sphinx-llms-txt: Found source file: {f.stem} at {f}")
             txt_files[f.stem] = f
 
+        # Log discovered files and page order
+        logger.debug(f"sphinx-llms-txt: Found {len(txt_files)} source files")
+        logger.debug(f"sphinx-llms-txt: Page order (after exclusion): {page_order}")
+
+        # Log exclusion patterns
+        exclude_patterns = self.config.get("llms_txt_exclude")
+        if exclude_patterns:
+            logger.debug(f"sphinx-llms-txt: Exclusion patterns: {exclude_patterns}")
+
         # Create a mapping from docnames to actual file names
         docname_to_file = {}
 
         # Try exact matches first
         for docname in page_order:
+            # Skip excluded pages
+            if any(
+                self._match_exclude_pattern(docname, pattern)
+                for pattern in exclude_patterns
+            ):
+                continue
+
             if docname in txt_files:
                 docname_to_file[docname] = txt_files[docname]
             else:
@@ -190,7 +219,26 @@ class LLMSFullManager:
                     abort_due_to_max_lines = True
                     break
 
-                if content:
+                # Double-check this file should be included (not in excluded patterns)
+                exclude_patterns = self.config.get("llms_txt_exclude")
+                file_stem = file_path.stem
+                should_include = True
+
+                if exclude_patterns:
+                    # Check stem and docname against exclusion patterns
+                    if any(
+                        self._match_exclude_pattern(file_stem, pattern)
+                        for pattern in exclude_patterns
+                    ) or any(
+                        self._match_exclude_pattern(docname, pattern)
+                        for pattern in exclude_patterns
+                    ):
+                        logger.debug(
+                            f"sphinx-llms-txt: Final exclusion check removed: {docname}"
+                        )
+                        should_include = False
+
+                if content and should_include:
                     content_parts.append(content)
                     added_files.add(file_path.stem)
                     total_line_count += line_count
@@ -199,8 +247,32 @@ class LLMSFullManager:
 
         # Add any remaining files (in alphabetical order) if not aborted
         if not abort_due_to_max_lines:
+            # Apply the same exclusion filter to remaining files
+            exclude_patterns = self.config.get("llms_txt_exclude")
+
+            # Create a set of files to exclude based on their basename
+            excluded_files = set()
+            for pattern in exclude_patterns:
+                if "*" not in pattern and "?" not in pattern:
+                    # For exact patterns, add variants
+                    excluded_files.add(pattern)
+                    excluded_files.add(f"{pattern}.rst")
+                    excluded_files.add(f"{pattern}.txt")
+                    excluded_files.add(pattern.replace("-", "_"))
+                    excluded_files.add(pattern.replace("_", "-"))
+
+            # Filter remaining files
             remaining_files = sorted(
-                [name for name in txt_files if name not in added_files]
+                [
+                    name
+                    for name in txt_files
+                    if name not in added_files
+                    and name not in excluded_files
+                    and not any(
+                        self._match_exclude_pattern(name, pattern)
+                        for pattern in exclude_patterns
+                    )
+                ]
             )
             if remaining_files:
                 logger.info(f"Adding remaining files: {remaining_files}")
@@ -212,7 +284,24 @@ class LLMSFullManager:
                 if max_lines is not None and total_line_count + line_count > max_lines:
                     break
 
-                if content:
+                # Double-check that this file should be included
+                should_include = True
+                file_stem = file_path.stem
+                exclude_patterns = self.config.get("llms_txt_exclude")
+
+                if exclude_patterns:
+                    # Check stem against exclusion patterns
+                    if any(
+                        self._match_exclude_pattern(file_stem, pattern)
+                        for pattern in exclude_patterns
+                    ):
+                        logger.debug(
+                            "sphinx-llms-txt: Final exclusion check removed remaining"
+                            f" file: {file_stem}"
+                        )
+                        should_include = False
+
+                if content and should_include:
                     content_parts.append(content)
                     total_line_count += line_count
 
@@ -260,7 +349,23 @@ class LLMSFullManager:
             tuple: (content_str, line_count) where line_count is the number of lines
                    in the file
         """
+        # Check if this file should be excluded by looking at the doc name
+        exclude_patterns = self.config.get("llms_txt_exclude")
+        if exclude_patterns and any(
+            self._match_exclude_pattern(docname, pattern)
+            for pattern in exclude_patterns
+        ):
+            return "", 0
+
         try:
+            # Check if the file stem (without extension) should be excluded
+            file_stem = file_path.stem
+            if exclude_patterns and any(
+                self._match_exclude_pattern(file_stem, pattern)
+                for pattern in exclude_patterns
+            ):
+                return "", 0
+
             with open(file_path, "r", encoding="utf-8") as f:
                 content = f.read()
 
@@ -483,6 +588,28 @@ class LLMSFullManager:
         processed_content = include_pattern.sub(replace_include, content)
         return processed_content
 
+    def _match_exclude_pattern(self, docname: str, pattern: str) -> bool:
+        """Check if a document name matches an exclude pattern.
+
+        Args:
+            docname: The document name to check
+            pattern: The pattern to match against
+
+        Returns:
+            True if the document should be excluded, False otherwise
+        """
+        # Exact match
+        if docname == pattern:
+            return True
+
+        # Glob-style pattern matching
+        import fnmatch
+
+        if fnmatch.fnmatch(docname, pattern):
+            return True
+
+        return False
+
     def _write_verbose_info_to_file(
         self, page_order: List[str], total_line_count: int = 0
     ):
@@ -561,6 +688,7 @@ def build_finished(app: Sphinx, exception):
             "llms_txt_full_filename": app.config.llms_txt_full_filename,
             "llms_txt_full_max_size": app.config.llms_txt_full_max_size,
             "llms_txt_directives": app.config.llms_txt_directives,
+            "llms_txt_exclude": app.config.llms_txt_exclude,
             "html_baseurl": getattr(app.config, "html_baseurl", ""),
         }
         _manager.set_config(config)
@@ -588,6 +716,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("llms_txt_directives", [], "env")
     app.add_config_value("llms_txt_title", None, "env")
     app.add_config_value("llms_txt_summary", None, "env")
+    app.add_config_value("llms_txt_exclude", [], "env")
 
     # Connect to Sphinx events
     app.connect("doctree-resolved", doctree_resolved)

tests/test_integration.py

@@ -162,3 +162,73 @@ def test_title_override(temp_dir, rootdir):
     # Safe unlink
     if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
         app.docutils_conf_path.unlink()
+
+
+def test_exclusion(temp_dir, rootdir):
+    """Test that the exclude patterns work correctly."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    # Create a new test app with exclude patterns
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_full_filename": "excluded.txt",
+            "llms_txt_exclude": [
+                "page1",
+                "page_with_*",
+            ],  # Exclude page1 and any page starting with page_with_
+        },
+    )
+
+    app.build()
+
+    # Check if the output file was created
+    output_file = Path(app.outdir) / "excluded.txt"
+    assert output_file.exists(), f"Output file {output_file} does not exist"
+
+    # Read the content of the output file
+    content = output_file.read_text()
+
+    # Check that index and page2 content is included
+    assert (
+        "Welcome to Test Project's documentation!" in content
+    )  # Index should be included
+    assert "Page 2 Title" in content  # page2 title should be included
+    assert "Content for section A" in content  # Content from page2 should be included
+
+    # Check that excluded content is NOT included
+    assert "Page 1 Title" not in content  # page1 title should be excluded
+    assert (
+        "Content for section 1" not in content
+    )  # Content from page1 should be excluded
+    assert (
+        "Page With Include" not in content
+    )  # page_with_include title should be excluded
+
+    # Extra debug info for test
+    print(f"\nContent snippet: {content[:500]}...\n")
+
+    # Check that none of the content from page1 appears
+    page1_phrases = [
+        "Page 1 Title",
+        "This is the content of page 1",
+        "Section 1",
+        "Content for section 1",
+        "Section 2",
+        "Content for section 2",
+    ]
+    for phrase in page1_phrases:
+        assert phrase not in content, f"Found excluded content: '{phrase}'"
+
+    # Custom cleanup to avoid missing_ok issue
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+
+    # Safe unlink
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()

tests/test_llms_txt.py

@@ -177,6 +177,23 @@ def test_process_includes_with_relative_paths(tmp_path):
     assert processed_content == expected_content
 
 
+def test_match_exclude_pattern():
+    """Test the _match_exclude_pattern method."""
+    # Create a manager
+    manager = LLMSFullManager()
+
+    # Test exact match
+    assert manager._match_exclude_pattern("page1", "page1") is True
+    assert manager._match_exclude_pattern("page1", "page2") is False
+
+    # Test glob-style patterns
+    assert manager._match_exclude_pattern("page1", "page*") is True
+    assert manager._match_exclude_pattern("page_with_include", "page_with_*") is True
+    assert manager._match_exclude_pattern("page1", "*1") is True
+    assert manager._match_exclude_pattern("subdir/page1", "*/page1") is True
+    assert manager._match_exclude_pattern("page1", "subdir/*") is False
+
+
 def test_write_verbose_info_to_file(tmp_path):
     """Test writing verbose info to a file."""
     # Create a manager