diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b2c2a8..9f7a4fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,13 +1,18 @@ Changelog ========= +0.2.1 +----- + +- Add ability to exclude pages with ``llms_txt_exclude`` + 0.2.0 ----- -- Add `llms_txt_full_max_size` configuration option to limit `llms-full.txt` file size -- Automatically add content from `include` directives in `llms-full.txt` -- Add path resolution for a given set of directives in `llms-full.txt` -- Add `llms.txt` file option, with `llms_txt_title` and `llms_txt_summary` config values +- Add ``llms_txt_full_max_size`` configuration option to limit `llms-full.txt` file size +- Automatically add content from **include** directives in **llms-full.txt** +- Add path resolution for a given set of directives in **llms-full.txt** +- Add **llms.txt** file option, with ``llms_txt_title`` and ``llms_txt_summary`` config values 0.1.0 ----- diff --git a/README.md b/README.md index 8243325..a630a92 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,12 @@ extensions = [ - **Default**: `None` - **Description**: Optional, but recommended, summary description for `llms.txt`. +### `llms_txt_exclude` + +- **Type**: list of strings +- **Default**: `[]` +- **Description**: A list of pages to ignore (e.g., "page1", "page_with_*"). + ## Features - Creates `llms.txt` and `llms-full.txt` @@ -76,6 +82,7 @@ extensions = [ - Resolves relative paths in directives like `image` and `figure` to use full paths - Ability to add list of custom directives with `llms_txt_directives` - Optionally, prepend a base URL using Sphinx's `html_baseurl` +- Ability to exclude pages ## License diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index 609da83..f627a9e 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -11,7 +11,7 @@ from sphinx.application import Sphinx from sphinx.environment import BuildEnvironment from sphinx.util import logging -__version__ = "0.2.0" +__version__ = "0.2.1" logger = logging.getLogger(__name__) @@ -122,6 +122,18 @@ class LLMSFullManager: ) return + # Apply exclusion filter if configured + exclude_patterns = self.config.get("llms_txt_exclude") + if exclude_patterns: + page_order = [ + page + for page in page_order + if not any( + self._match_exclude_pattern(page, pattern) + for pattern in exclude_patterns + ) + ] + # Determine output file name and location output_filename = self.config.get("llms_txt_full_filename") output_path = Path(outdir) / output_filename @@ -148,13 +160,30 @@ class LLMSFullManager: # Collect all available source files txt_files = {} for f in sources_dir.glob("*.txt"): + logger.debug(f"sphinx-llms-txt: Found source file: {f.stem} at {f}") txt_files[f.stem] = f + # Log discovered files and page order + logger.debug(f"sphinx-llms-txt: Found {len(txt_files)} source files") + logger.debug(f"sphinx-llms-txt: Page order (after exclusion): {page_order}") + + # Log exclusion patterns + exclude_patterns = self.config.get("llms_txt_exclude") + if exclude_patterns: + logger.debug(f"sphinx-llms-txt: Exclusion patterns: {exclude_patterns}") + # Create a mapping from docnames to actual file names docname_to_file = {} # Try exact matches first for docname in page_order: + # Skip excluded pages + if any( + self._match_exclude_pattern(docname, pattern) + for pattern in exclude_patterns + ): + continue + if docname in txt_files: docname_to_file[docname] = txt_files[docname] else: @@ -190,7 +219,26 @@ class LLMSFullManager: abort_due_to_max_lines = True break - if content: + # Double-check this file should be included (not in excluded patterns) + exclude_patterns = self.config.get("llms_txt_exclude") + file_stem = file_path.stem + should_include = True + + if exclude_patterns: + # Check stem and docname against exclusion patterns + if any( + self._match_exclude_pattern(file_stem, pattern) + for pattern in exclude_patterns + ) or any( + self._match_exclude_pattern(docname, pattern) + for pattern in exclude_patterns + ): + logger.debug( + f"sphinx-llms-txt: Final exclusion check removed: {docname}" + ) + should_include = False + + if content and should_include: content_parts.append(content) added_files.add(file_path.stem) total_line_count += line_count @@ -199,8 +247,32 @@ class LLMSFullManager: # Add any remaining files (in alphabetical order) if not aborted if not abort_due_to_max_lines: + # Apply the same exclusion filter to remaining files + exclude_patterns = self.config.get("llms_txt_exclude") + + # Create a set of files to exclude based on their basename + excluded_files = set() + for pattern in exclude_patterns: + if "*" not in pattern and "?" not in pattern: + # For exact patterns, add variants + excluded_files.add(pattern) + excluded_files.add(f"{pattern}.rst") + excluded_files.add(f"{pattern}.txt") + excluded_files.add(pattern.replace("-", "_")) + excluded_files.add(pattern.replace("_", "-")) + + # Filter remaining files remaining_files = sorted( - [name for name in txt_files if name not in added_files] + [ + name + for name in txt_files + if name not in added_files + and name not in excluded_files + and not any( + self._match_exclude_pattern(name, pattern) + for pattern in exclude_patterns + ) + ] ) if remaining_files: logger.info(f"Adding remaining files: {remaining_files}") @@ -212,7 +284,24 @@ class LLMSFullManager: if max_lines is not None and total_line_count + line_count > max_lines: break - if content: + # Double-check that this file should be included + should_include = True + file_stem = file_path.stem + exclude_patterns = self.config.get("llms_txt_exclude") + + if exclude_patterns: + # Check stem against exclusion patterns + if any( + self._match_exclude_pattern(file_stem, pattern) + for pattern in exclude_patterns + ): + logger.debug( + "sphinx-llms-txt: Final exclusion check removed remaining" + f" file: {file_stem}" + ) + should_include = False + + if content and should_include: content_parts.append(content) total_line_count += line_count @@ -260,7 +349,23 @@ class LLMSFullManager: tuple: (content_str, line_count) where line_count is the number of lines in the file """ + # Check if this file should be excluded by looking at the doc name + exclude_patterns = self.config.get("llms_txt_exclude") + if exclude_patterns and any( + self._match_exclude_pattern(docname, pattern) + for pattern in exclude_patterns + ): + return "", 0 + try: + # Check if the file stem (without extension) should be excluded + file_stem = file_path.stem + if exclude_patterns and any( + self._match_exclude_pattern(file_stem, pattern) + for pattern in exclude_patterns + ): + return "", 0 + with open(file_path, "r", encoding="utf-8") as f: content = f.read() @@ -483,6 +588,28 @@ class LLMSFullManager: processed_content = include_pattern.sub(replace_include, content) return processed_content + def _match_exclude_pattern(self, docname: str, pattern: str) -> bool: + """Check if a document name matches an exclude pattern. + + Args: + docname: The document name to check + pattern: The pattern to match against + + Returns: + True if the document should be excluded, False otherwise + """ + # Exact match + if docname == pattern: + return True + + # Glob-style pattern matching + import fnmatch + + if fnmatch.fnmatch(docname, pattern): + return True + + return False + def _write_verbose_info_to_file( self, page_order: List[str], total_line_count: int = 0 ): @@ -561,6 +688,7 @@ def build_finished(app: Sphinx, exception): "llms_txt_full_filename": app.config.llms_txt_full_filename, "llms_txt_full_max_size": app.config.llms_txt_full_max_size, "llms_txt_directives": app.config.llms_txt_directives, + "llms_txt_exclude": app.config.llms_txt_exclude, "html_baseurl": getattr(app.config, "html_baseurl", ""), } _manager.set_config(config) @@ -588,6 +716,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_config_value("llms_txt_directives", [], "env") app.add_config_value("llms_txt_title", None, "env") app.add_config_value("llms_txt_summary", None, "env") + app.add_config_value("llms_txt_exclude", [], "env") # Connect to Sphinx events app.connect("doctree-resolved", doctree_resolved) diff --git a/tests/test_integration.py b/tests/test_integration.py index 6a3350d..c945d3a 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -162,3 +162,73 @@ def test_title_override(temp_dir, rootdir): # Safe unlink if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): app.docutils_conf_path.unlink() + + +def test_exclusion(temp_dir, rootdir): + """Test that the exclude patterns work correctly.""" + from sphinx.testing.util import SphinxTestApp + + src_dir = rootdir / "basic" + + # Create a new test app with exclude patterns + app = SphinxTestApp( + srcdir=src_dir, + builddir=temp_dir, + buildername="html", + freshenv=True, + confoverrides={ + "llms_txt_full_filename": "excluded.txt", + "llms_txt_exclude": [ + "page1", + "page_with_*", + ], # Exclude page1 and any page starting with page_with_ + }, + ) + + app.build() + + # Check if the output file was created + output_file = Path(app.outdir) / "excluded.txt" + assert output_file.exists(), f"Output file {output_file} does not exist" + + # Read the content of the output file + content = output_file.read_text() + + # Check that index and page2 content is included + assert ( + "Welcome to Test Project's documentation!" in content + ) # Index should be included + assert "Page 2 Title" in content # page2 title should be included + assert "Content for section A" in content # Content from page2 should be included + + # Check that excluded content is NOT included + assert "Page 1 Title" not in content # page1 title should be excluded + assert ( + "Content for section 1" not in content + ) # Content from page1 should be excluded + assert ( + "Page With Include" not in content + ) # page_with_include title should be excluded + + # Extra debug info for test + print(f"\nContent snippet: {content[:500]}...\n") + + # Check that none of the content from page1 appears + page1_phrases = [ + "Page 1 Title", + "This is the content of page 1", + "Section 1", + "Content for section 1", + "Section 2", + "Content for section 2", + ] + for phrase in page1_phrases: + assert phrase not in content, f"Found excluded content: '{phrase}'" + + # Custom cleanup to avoid missing_ok issue + sys.path[:] = app._saved_path + _clean_up_global_state() + + # Safe unlink + if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists(): + app.docutils_conf_path.unlink() diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index d2f94ed..9a8fd3c 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -177,6 +177,23 @@ def test_process_includes_with_relative_paths(tmp_path): assert processed_content == expected_content +def test_match_exclude_pattern(): + """Test the _match_exclude_pattern method.""" + # Create a manager + manager = LLMSFullManager() + + # Test exact match + assert manager._match_exclude_pattern("page1", "page1") is True + assert manager._match_exclude_pattern("page1", "page2") is False + + # Test glob-style patterns + assert manager._match_exclude_pattern("page1", "page*") is True + assert manager._match_exclude_pattern("page_with_include", "page_with_*") is True + assert manager._match_exclude_pattern("page1", "*1") is True + assert manager._match_exclude_pattern("subdir/page1", "*/page1") is True + assert manager._match_exclude_pattern("page1", "subdir/*") is False + + def test_write_verbose_info_to_file(tmp_path): """Test writing verbose info to a file.""" # Create a manager