Add llms.txt config options (#9)

CHANGELOG.rst

@@ -4,9 +4,10 @@ Changelog
 0.2.0
 -----
 
-- Add `llms_txt_max_lines` configuration option to limit `llms-full.txt` file size
+- Add `llms_txt_full_max_size` configuration option to limit `llms-full.txt` file size
-- Automatically add content from `.. include::` directives
+- Automatically add content from `include` directives in  `llms-full.txt`
-- Add path resolution for certain directives
+- Add path resolution for a given set of directives  in `llms-full.txt`
+- Add `llms.txt` file option, with `llms_txt_title` and `llms_txt_summary` config values
 
 0.1.0
 -----

README.md

@@ -1,6 +1,6 @@
-# Sphinx llms-full.txt Extension
+# Sphinx llms.txt generator
 
-A Sphinx extension that creates a single combined documentation `llms-full.txt` file, written in reStructuredText.
+A Sphinx extension that generates a summary `llms.txt` file, written in Markdown, and a single combined documentation `llms-full.txt` file, written in reStructuredText.
 
 ## Installation
 
@@ -20,23 +20,36 @@ extensions = [
 
 ## Configuration Options
 
-### `llms_txt_filename`
+### `llms_txt_full_file`
+
+- **Type**: boolean
+- **Default**: `'True`
+- **Description**: Whether to write the single output file
+
+### `llms_txt_full_filename`
 
 - **Type**: string
 - **Default**: `'llms-full.txt'`
-- **Description**: Name of the output file
+- **Description**: Name of the single output file
 
-### `llms_txt_verbose`
+### `llms_txt_full_max_size`
-
-- **Type**: boolean
-- **Default**: `False`
-- **Description**: Whether to include a summary in the build output
-
-### `llms_txt_max_lines`
 
 - **Type**: integer or `None`
 - **Default**: `None` (no limit)
-- **Description**: Sets a maximum line count for `llms_txt_filename`. If exceeded, the file is skipped and a warning is shown, but the build still completes.
+- **Description**: Sets a maximum line count for `llms_txt_full_filename`.
+  If exceeded, the file is skipped and a warning is shown, but the build still completes.
+
+### `llms_txt_file`
+
+- **Type**: boolean
+- **Default**: `True`
+- **Description**: Whether to write the summary information file
+
+### `llms_txt_filename`
+
+- **Type**: string
+- **Default**: `llms.txt`
+- **Description**: Name of the summary information file
 
 ### `llms_txt_directives`
 
@@ -44,8 +57,21 @@ extensions = [
 - **Default**: `[]` (empty list)
 - **Description**: List of custom directive names to process for path resolution.
 
+### `llms_txt_title`
+
+- **Type**: string or `None`
+- **Default**: `None`
+- **Description**: Overrides the Sphinx project name as the heading in `llms.txt`.
+
+### `llms_txt_summary`
+
+- **Type**: string or `None`
+- **Default**: `None`
+- **Description**: Optional, but recommended, summary description for `llms.txt`.
+
 ## Features
 
+- Creates `llms.txt` and `llms-full.txt`
 - Automatically add content from `include` directives
 - Resolves relative paths in directives like `image` and `figure` to use full paths
   - Ability to add list of custom directives with `llms_txt_directives`

sphinx_llms_txt/__init__.py

@@ -26,6 +26,7 @@ class LLMSFullManager:
         self.env: BuildEnvironment = None
         self.srcdir: Optional[str] = None
         self.outdir: Optional[str] = None
+        self.app: Optional[Sphinx] = None
 
     def set_master_doc(self, master_doc: str):
         """Set the master document name."""
@@ -44,6 +45,10 @@ class LLMSFullManager:
         """Set configuration options."""
         self.config = config
 
+    def set_app(self, app: Sphinx):
+        """Set the Sphinx application reference."""
+        self.app = app
+
     def get_page_order(self) -> List[str]:
         """Get the correct page order from the toctree structure."""
         if not self.env or not self.master_doc:
@@ -118,7 +123,7 @@ class LLMSFullManager:
             return
 
         # Determine output file name and location
-        output_filename = self.config.get("llms_txt_filename")
+        output_filename = self.config.get("llms_txt_full_filename")
         output_path = Path(outdir) / output_filename
 
         # Find sources directory
@@ -172,7 +177,7 @@ class LLMSFullManager:
         # Add pages in order
         added_files = set()
         total_line_count = 0
-        max_lines = self.config.get("llms_txt_max_lines")
+        max_lines = self.config.get("llms_txt_full_max_size")
         abort_due_to_max_lines = False
 
         for docname in page_order:
@@ -212,7 +217,7 @@ class LLMSFullManager:
                     total_line_count += line_count
 
         # Check if line limit was exceeded before creating the file
-        max_lines = self.config.get("llms_txt_max_lines")
+        max_lines = self.config.get("llms_txt_full_max_size")
         if abort_due_to_max_lines or (
             max_lines is not None and total_line_count > max_lines
         ):
@@ -223,8 +228,8 @@ class LLMSFullManager:
             )
 
             # Log summary information if requested
-            if self.config.get("llms_txt_verbose"):
+            if self.config.get("llms_txt_file"):
-                self._log_summary_info(page_order, total_line_count)
+                self._write_verbose_info_to_file(page_order, total_line_count)
 
             return
 
@@ -239,8 +244,8 @@ class LLMSFullManager:
             )
 
             # Log summary information if requested
-            if self.config.get("llms_txt_verbose"):
+            if self.config.get("llms_txt_file"):
-                self._log_summary_info(page_order, total_line_count)
+                self._write_verbose_info_to_file(page_order, total_line_count)
 
         except Exception as e:
             logger.error(f"sphinx-llm-txt: Error writing combined sources file: {e}")
@@ -478,18 +483,45 @@ class LLMSFullManager:
         processed_content = include_pattern.sub(replace_include, content)
         return processed_content
 
-    def _log_summary_info(self, page_order: List[str], total_line_count: int = 0):
+    def _write_verbose_info_to_file(
-        """Log summary information to the logger."""
+        self, page_order: List[str], total_line_count: int = 0
-        logger.info("")
+    ):
-        logger.info("llms-txt Summary")
+        """Write summary information to the llms.txt file."""
-        logger.info("================")
+        if not self.outdir:
-        logger.info(f"Total pages: {len(page_order)}")
+            logger.warning(
+                "sphinx-llms-txt: Cannot write verbose info to file: outdir not set"
+            )
+            return
 
-        logger.info(f"Configuration: {self.config}")
+        output_path = Path(self.outdir) / self.config.get("llms_txt_filename")
-        logger.info("Page order:")
+        try:
+            with open(output_path, "w", encoding="utf-8") as f:
+                project_name = "llms-txt Summary"
+                # First priority: use title from config if available
+                if self.config.get("llms_txt_title"):
+                    project_name = self.config.get("llms_txt_title")
+                # Second priority: use project name from Sphinx app if available
+                elif (
+                    self.app
+                    and hasattr(self.app, "config")
+                    and hasattr(self.app.config, "project")
+                ):
+                    project_name = self.app.config.project
+                f.write(f"# {project_name}\n\n")
+
+                # Add description if available
+                description = self.config.get("llms_txt_summary", "")
+                if description:
+                    f.write(f"> {description}\n\n")
+
+                f.write("## Docs\n\n")
                 for i, docname in enumerate(page_order, 1):
                     title = self.page_titles.get(docname, docname)
-            logger.info(f"{i:3d}. {docname} - {title}")
+                    f.write(f"- [{title}](/{docname}.html)\n")
+
+            logger.info(f"sphinx-llms-txt: created {output_path}")
+        except Exception as e:
+            logger.error(f"sphinx-llms-txt: Error writing verbose info to file: {e}")
 
 
 # Global manager instance
@@ -517,12 +549,17 @@ def build_finished(app: Sphinx, exception):
         # Set the environment and master doc in the manager
         _manager.set_env(app.env)
         _manager.set_master_doc(app.config.master_doc)
+        _manager.set_app(app)
 
         # Set up configuration
         config = {
+            "llms_txt_file": app.config.llms_txt_file,
             "llms_txt_filename": app.config.llms_txt_filename,
-            "llms_txt_verbose": app.config.llms_txt_verbose,
+            "llms_txt_title": app.config.llms_txt_title,
-            "llms_txt_max_lines": app.config.llms_txt_max_lines,
+            "llms_txt_summary": app.config.llms_txt_summary,
+            "llms_txt_full_file": app.config.llms_txt_full_file,
+            "llms_txt_full_filename": app.config.llms_txt_full_filename,
+            "llms_txt_full_max_size": app.config.llms_txt_full_max_size,
             "llms_txt_directives": app.config.llms_txt_directives,
             "html_baseurl": getattr(app.config, "html_baseurl", ""),
         }
@@ -543,10 +580,14 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     """Set up the Sphinx extension."""
 
     # Add configuration options
-    app.add_config_value("llms_txt_filename", "llms-full.txt", "env")
+    app.add_config_value("llms_txt_file", True, "env")
-    app.add_config_value("llms_txt_verbose", False, "env")
+    app.add_config_value("llms_txt_filename", "llms.txt", "env")
-    app.add_config_value("llms_txt_max_lines", None, "env")
+    app.add_config_value("llms_txt_full_file", True, "env")
+    app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
+    app.add_config_value("llms_txt_full_max_size", None, "env")
     app.add_config_value("llms_txt_directives", [], "env")
+    app.add_config_value("llms_txt_title", None, "env")
+    app.add_config_value("llms_txt_summary", None, "env")
 
     # Connect to Sphinx events
     app.connect("doctree-resolved", doctree_resolved)

tests/roots/basic/conf.py

@@ -15,8 +15,8 @@ html_theme = "alabaster"
 html_static_path = ["_static"]
 
 # Configuration for sphinx-llms-txt
-llms_txt_filename = "test-llms-full.txt"
+llms_txt_full_filename = "test-llms-full.txt"
-llms_txt_verbose = True
+llms_txt_file = True
 
 # Master document
 master_doc = "index"

tests/roots/basic/conf_custom.py

@@ -15,8 +15,8 @@ html_theme = "alabaster"
 html_static_path = ["_static"]
 
 # Configuration for sphinx-llms-txt
-llms_txt_filename = "custom-name.txt"
+llms_txt_full_filename = "custom-name.txt"
-llms_txt_verbose = True
+llms_txt_file = True
 
 # Master document
 master_doc = "index"

tests/test_integration.py

@@ -39,6 +39,7 @@ def test_custom_filename(temp_dir, rootdir):
     from sphinx.testing.util import SphinxTestApp
 
     src_dir = rootdir / "basic"
+    print(src_dir)
 
     # Create a copy of the configuration with a different filename
     custom_conf = src_dir / "conf_custom.py"
@@ -46,8 +47,8 @@ def test_custom_filename(temp_dir, rootdir):
         conf_content = f.read()
 
     conf_content = conf_content.replace(
-        'llms_txt_filename = "test-llms-full.txt"',
+        'llms_txt_full_filename = "test-llms-full.txt"',
-        'llms_txt_filename = "custom-name.txt"',
+        'llms_txt_full_filename = "custom-name.txt"',
     )
 
     with open(custom_conf, "w") as f:
@@ -59,7 +60,7 @@ def test_custom_filename(temp_dir, rootdir):
         builddir=temp_dir,
         buildername="html",
         freshenv=True,
-        confoverrides={"llms_txt_filename": "custom-name.txt"},
+        confoverrides={"llms_txt_full_filename": "custom-name.txt"},
     )
 
     app.build()
@@ -94,9 +95,8 @@ def test_max_lines_limit(temp_dir, rootdir):
         buildername="html",
         freshenv=True,
         confoverrides={
-            "llms_txt_filename": "limited.txt",
+            "llms_txt_full_filename": "limited.txt",
-            "llms_txt_max_lines": 10,  # Set a small limit to trigger the warning
+            "llms_txt_full_max_size": 10,  # Set a small limit to trigger the warning
-            "llms_txt_verbose": True,
         },
     )
 
@@ -115,3 +115,50 @@ def test_max_lines_limit(temp_dir, rootdir):
     # Safe unlink
     if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
         app.docutils_conf_path.unlink()
+
+
+def test_title_override(temp_dir, rootdir):
+    """Test that the title override works correctly."""
+    from sphinx.testing.util import SphinxTestApp
+
+    src_dir = rootdir / "basic"
+
+    # Custom title to override the default project name
+    custom_title = "Custom Title Override"
+
+    # Create a new test app with the title override
+    app = SphinxTestApp(
+        srcdir=src_dir,
+        builddir=temp_dir,
+        buildername="html",
+        freshenv=True,
+        confoverrides={
+            "llms_txt_title": custom_title,
+        },
+    )
+
+    app.build()
+
+    # Check if the summary file was created
+    summary_file = Path(app.outdir) / "llms.txt"
+    assert summary_file.exists(), f"Summary file {summary_file} does not exist"
+
+    # Read the content of the summary file
+    content = summary_file.read_text()
+
+    # Check that the custom title was used
+    assert (
+        f"# {custom_title}" in content
+    ), f"Custom title '{custom_title}' not found in summary file"
+    # Ensure the default project name was NOT used
+    assert (
+        "# Test Project" not in content
+    ), "Default project name was used instead of custom title"
+
+    # Custom cleanup to avoid missing_ok issue
+    sys.path[:] = app._saved_path
+    _clean_up_global_state()
+
+    # Safe unlink
+    if hasattr(app, "docutils_conf_path") and app.docutils_conf_path.exists():
+        app.docutils_conf_path.unlink()

tests/test_llms_txt.py

@@ -58,9 +58,9 @@ def test_set_config():
     """Test setting configuration."""
     manager = LLMSFullManager()
     config = {
-        "llms_txt_filename": "custom.txt",
+        "llms_txt_full_filename": "custom.txt",
-        "llms_txt_verbose": True,
+        "llms_txt_file": True,
-        "llms_txt_max_lines": 1000,
+        "llms_txt_full_max_size": 1000,
     }
     manager.set_config(config)
     assert manager.config == config
@@ -175,3 +175,47 @@ def test_process_includes_with_relative_paths(tmp_path):
         " directory.\nLine after include."
     )
     assert processed_content == expected_content
+
+
+def test_write_verbose_info_to_file(tmp_path):
+    """Test writing verbose info to a file."""
+    # Create a manager
+    manager = LLMSFullManager()
+
+    # Set up a build directory
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    # Set the outdir on the manager
+    manager.outdir = str(build_dir)
+
+    # Set configuration with verbose_file enabled
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_full_max_size": 1000,
+        "llms_txt_filename": "llms.txt",
+    }
+    manager.set_config(config)
+
+    # Add some page titles
+    manager.update_page_title("index", "Home Page")
+    manager.update_page_title("about", "About Us")
+
+    # Create a page order
+    page_order = ["index", "about"]
+
+    # Call the method to write verbose info to file
+    manager._write_verbose_info_to_file(page_order, 500)
+
+    # Check that the file was created
+    verbose_file = build_dir / "llms.txt"
+    assert verbose_file.exists()
+
+    # Read the file content
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Check that the content contains expected information
+    assert "## Docs" in content
+    assert "- [Home Page](/index.html)" in content
+    assert "- [About Us](/about.html)" in content