Merge branch 'main' into feature/singlemarkdown

.readthedocs.yml

@@ -9,10 +9,9 @@ build:
     - pip install -r docs/requirements.txt
     - pip install -e .
     - cmake --workflow --preset documentation-workflow
-    # Generate llms.txt variants for demo purposes
-    - python docs/generate_llms_variants.py build/html
     # Copy built documentation to Read the Docs output directory
     - mkdir -p $READTHEDOCS_OUTPUT/html
     - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
     - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
+    - cp build/singlemarkdown/index.md $READTHEDOCS_OUTPUT/html/llms-full.txt
     - cp -r build/rst/* $READTHEDOCS_OUTPUT/html/

CHANGELOG.rst

@@ -1,11 +1,6 @@
 Changelog
 =========
 
-0.7.1
------
-
-- Don't process includes within code blocks
-
 0.7.0
 -----
 

CMakeLists.txt

@@ -1,15 +1,8 @@
 cmake_minimum_required(VERSION 3.15)
-project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
+project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
 
-# Fetch Sphinx CMake modules
+# Add CMake module path
-include(FetchContent)
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-FetchContent_Declare(
-  sphinx_cmake_modules
-  GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
-  GIT_TAG        main
-)
-FetchContent_MakeAvailable(sphinx_cmake_modules)
-list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
 
 # Add documentation
 add_subdirectory(docs)

CMakePresets.json

@@ -21,6 +21,12 @@
       "configurePreset": "documentation",
       "targets": ["markdown"]
     },
+    {
+      "name": "singlemarkdown",
+      "displayName": "Build Single Markdown Documentation",
+      "configurePreset": "documentation",
+      "targets": ["singlemarkdown"]
+    },
     {
       "name": "rst",
       "displayName": "Build reStructuredText Documentation",
@@ -29,9 +35,9 @@
     },
     {
       "name": "docs-parallel",
-      "displayName": "Build all output formats in parallel",
+      "displayName": "Build HTML and Markdown in parallel",
       "configurePreset": "documentation",
-      "targets": ["html", "markdown", "rst"]
+      "targets": ["html", "markdown", "singlemarkdown", "rst"]
     }
   ],
   "workflowPresets": [

cmake/SphinxUtils.cmake

@@ -0,0 +1,28 @@
+# Sphinx related utilities
+
+set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
+
+# Function to find Sphinx in the system
+function(setup_sphinx_environment)
+    # Find sphinx-build executable in system
+    find_program(SPHINX_EXECUTABLE
+        NAMES sphinx-build
+        DOC "Sphinx documentation generator"
+    )
+
+    if(NOT SPHINX_EXECUTABLE)
+        message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
+    endif()
+
+    # Export to parent scope
+    set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
+endfunction()
+
+# Function to add a Sphinx builder target
+function(add_sphinx_builder builder_name)
+    add_custom_target(${builder_name}
+        COMMAND ${SPHINX_EXECUTABLE} -b ${builder_name} ${SPHINX_SOURCE} ${SPHINX_BUILD}/${builder_name}
+        VERBATIM
+    )
+endfunction()

docs/CMakeLists.txt

@@ -4,4 +4,5 @@ setup_sphinx_environment()
 
 add_sphinx_builder(html)
 add_sphinx_builder(markdown)
+add_sphinx_builder(singlemarkdown)
 add_sphinx_builder(rst)

docs/generate_llms_variants.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-"""
-Generate variant llms.txt files for demo purposes.
-
-Takes the generated llms.txt (with _sources links) and creates:
-- llms.txt - default with _sources links (unchanged)
-- llms.md.txt - .html.md links
-- llms.rst.txt - .rst links
-"""
-
-import re
-import sys
-from pathlib import Path
-
-
-def get_base_url() -> str:
-    """Import base_url from conf.py."""
-    sys.path.insert(0, str(Path(__file__).parent / "source"))
-    from conf import html_baseurl  # noqa: E402
-
-    return html_baseurl
-
-
-def generate_variants(build_dir: Path) -> None:
-    """Generate llms.txt variants from the original file."""
-    original = build_dir / "llms.txt"
-
-    if not original.exists():
-        print(f"Error: {original} not found")
-        sys.exit(1)
-
-    content = original.read_text()
-    base_url = get_base_url()
-
-    # Pattern to match links like: https://.../_sources/{docname}.rst.txt
-    link_pattern = re.compile(
-        rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
-    )
-
-    # Generate .html.md variant
-    md_content = link_pattern.sub(r"\1\2.html.md", content)
-    (build_dir / "llms.md.txt").write_text(md_content)
-    print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
-
-    # Generate .rst variant
-    rst_content = link_pattern.sub(r"\1\2.rst", content)
-    (build_dir / "llms.rst.txt").write_text(rst_content)
-    print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
-
-    print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
-
-
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        build_dir = Path(sys.argv[1])
-    else:
-        build_dir = Path("build/html")
-
-    generate_variants(build_dir)

docs/requirements.txt

@@ -2,9 +2,8 @@ furo
 esbonio
 sphinx-contributors
 sphinx
-sphinx-design
 sphinx-llms-txt
 sphinx-inline-tabs
 sphinxext-opengraph
-sphinx-markdown-builder
+sphinx-markdown-builder @ git+https://github.com/akaihola/sphinx-markdown-builder.git@singlemarkdown
 sphinxcontrib-restbuilder

docs/source/advanced-configuration.rst

@@ -294,145 +294,9 @@ Your URI template can use the following variables:
 
 .. tip::
    Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
-   See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
+   See this package's `CMake setup <https://github.com/jdillard/sphinx-llms-txt>`_ for an example of building both HTML and Markdown and/or reStructuredText in parallel.
    Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
 
-.. _cmake_workflow:
-
-CMake Workflow
-^^^^^^^^^^^^^^
-
-This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
-This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
-
-Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
-Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
-
-Key Files
-~~~~~~~~~
-
-These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
-
-.. code-block:: text
-
-   .
-   ├── .readthedocs.yml
-   ├── CMakeLists.txt
-   ├── CMakePresets.json
-   └── docs/
-       └── CMakeLists.txt
-
-Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
-
-.. dropdown:: .readthedocs.yml
-   :chevron: down-up
-
-   A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
-
-   .. literalinclude:: ../../.readthedocs.yml
-      :language: yaml
-      :lines: 1-9,11,14-
-      :linenos:
-      :emphasize-lines: 9, 14-15
-
-   .. list-table::
-      :header-rows: 1
-      :width: 100%
-      :widths: 15 85
-
-      * - Line
-        - Description
-      * - **9**
-        - Update the path if your requirements file is in a different location
-      * - **13-14**
-        - Modify the copy commands for the output formats you deploy
-
-.. dropdown:: CMakeLists.txt
-   :chevron: down-up
-
-   A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
-
-   .. literalinclude:: ../../CMakeLists.txt
-      :language: cmake
-      :linenos:
-      :emphasize-lines: 9, 15
-
-   .. list-table::
-      :header-rows: 1
-      :width: 100%
-      :widths: 15 85
-
-      * - Line
-        - Description
-      * - **9**
-        - Update the ``GIT_TAG`` to use a different version or commit hash
-      * - **15**
-        - Change if your docs subdirectory has a different location
-
-.. dropdown:: docs/CMakeLists.txt
-   :chevron: down-up
-
-   A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
-
-   .. literalinclude:: ../CMakeLists.txt
-      :language: cmake
-      :linenos:
-      :emphasize-lines: 5-7
-
-   .. list-table::
-      :header-rows: 1
-      :width: 100%
-      :widths: 15 85
-
-      * - Line
-        - Description
-      * - **5-7**
-        - Add or remove calls based on which output formats you need
-
-
-.. dropdown:: CMakePresets.json
-   :chevron: down-up
-
-   Defines presets for configuring and building documentation:
-
-   - **Configure Presets:** Sets up the build directory.
-   - **Build Presets:** Defines build formats individually and all in parallel.
-   - **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
-
-   .. literalinclude:: ../../CMakePresets.json
-      :language: json
-      :linenos:
-      :emphasize-lines: 18-23, 24-29, 34
-
-   .. list-table::
-      :header-rows: 1
-      :width: 100%
-      :widths: 15 85
-
-      * - Line
-        - Description
-      * - **18-23**
-        - Remove this preset to disable Markdown documentation builds
-      * - **24-29**
-        - Remove this preset to disable reStructuredText documentation builds
-      * - **34**
-        - Modify the targets list to build only the output formats you need in parallel
-
-Usage
-~~~~~
-
-To build documentation locally using CMake:
-
-.. code-block:: console
-
-   # Run the full workflow (configure + build all formats)
-   cmake --workflow --preset documentation-workflow
-
-   # Or configure and build separately
-   cmake --preset documentation
-   cmake --build --preset html        # Build HTML only
-   cmake --build --preset docs-parallel  # Build all formats
-
 .. _integration_examples:
 
 Integration Examples

docs/source/conf.py

@@ -16,6 +16,8 @@ project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
 
+llms_txt_uri_template = "{base_url}{docname}.md"
+llms_txt_full_file = False
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
 A Sphinx extension that generates a summary llms.txt file,written in Markdown,
@@ -24,7 +26,6 @@ and a single combined documentation llms-full.txt file, written in reStructuredT
 
 # This doesn't seem to be supported
 # rst_file_suffix = ".html.rst"
-markdown_file_suffix = ".html.md"
 
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
@@ -56,8 +57,6 @@ extensions = [
     "sphinx_llms_txt",
     "sphinxcontrib.restbuilder",
     "sphinx_inline_tabs",
-    "sphinx.ext.extlinks",
-    "sphinx_design",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/source/getting-started.rst

@@ -34,59 +34,7 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
     sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
     sphinx-llms-txt: created /path/to/_build/html/llms.txt
 
-.. _choosing-output-format:
 
-Choosing an Output Format
+.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades.
--------------------------
-
-By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
-For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
-
-.. list-table:: Output Format Comparison
-   :header-rows: 1
-   :widths: 18 27 27 27
-
-   * -
-     - Default
-     - Markdown
-     - reStructuredText
-   * - **Setup**
-     - No config
-     - CMake [#sphinxllm]_
-     - CMake
-   * - **Builder**
-     - Native [#native]_
-     - `sphinx-markdown-builder`_
-     - `sphinxcontrib-restbuilder`_
-   * - **Format**
-     - Raw reStructuredText source
-     - Rendered Markdown [#rendered]_
-     - Rendered reStructuredText [#rendered]_
-   * - **LLM Readability**
-     - Good - preserves structure for simple syntax
-     - Excellent - native LLM format
-     - Good - Can provide more structured content
-   * - **Key Advantage**
-     - Zero setup required
-     - More compact (less input tokens)
-     - Can preserve Sphinx semantics
-   * - **Key Disadvantage**
-     - Raw directives won't be parsed [#autodoc]_
-     - Loses structure from complex directives
-     - Can lose structure from complex directives
-   * - **llms-full.txt support**
-     - Supported with above caveats
-     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
-     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
-
-.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
-.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
-
-.. rubric:: Footnotes
-
-.. [#sphinxllm] See `sphinx-llm <https://github.com/jacobtomlinson/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
-.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
-.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
-.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
-.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
 
+See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.

docs/source/index.rst

@@ -8,23 +8,21 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
 Demo
 ----
 
-This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
+You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example.
-
-Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
 
 Highlights
 ----------
 
-**Zero Configuration**
+1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build
-   Add the extension to your ``conf.py`` and you're done.
+2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content
-   The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
+3. **Path Resolution**: Transforms relative paths in directives to full paths
+4. **Output Generation**: Creates two optional files:
 
-**Intelligent Content Processing**
+   - ``llms.txt``: A concise summary of your documentation, in Markdown
-   Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
+   - ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText
 
-**Customizable When Needed**
+5. **Content Filtering**: Allows you to exclude specific pages or sections
-   Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
+6. **Source Code**: Allows you to include specific source code files
-   See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
 
 .. toctree::
    :maxdepth: 2
@@ -38,8 +36,6 @@ Highlights
 
 .. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
 .. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
-.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
-.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
 .. _Sphinx: http://sphinx-doc.org/
 
 .. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg

sphinx_llms_txt/__init__.py

@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.7.1"
+__version__ = "0.7.0"
 
 # Export classes needed by tests
 __all__ = [

sphinx_llms_txt/processor.py

@@ -128,11 +128,8 @@ class DocumentProcessor:
         Returns:
             Processed content with directive paths properly resolved
         """
-        # Get code block ranges to skip directives inside them
-        code_block_ranges = self._get_code_block_ranges(content)
-
         # Get the configured path directives to process
-        default_path_directives = ["image", "figure", "literalinclude"]
+        default_path_directives = ["image", "figure"]
         custom_path_directives = self.config.get("llms_txt_directives")
         path_directives = set(default_path_directives + custom_path_directives)
 
@@ -146,11 +143,6 @@ class DocumentProcessor:
         is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
 
         def replace_directive_path(match, base_url=base_url, is_test=is_test):
-            # Check if this directive is within a code block
-            if self._is_in_code_block(match.start(), code_block_ranges):
-                # This directive is inside a code block, don't process it
-                return match.group(0)
-
             prefix = match.group(1)  # The entire directive prefix including whitespace
             path = match.group(3).strip()  # The path argument
 
@@ -284,71 +276,6 @@ class DocumentProcessor:
 
         return possible_paths
 
-    def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
-        """Find all code block ranges in the content.
-
-        Args:
-            content: The source content to analyze
-
-        Returns:
-            List of (start, end) tuples representing code block character
-            ranges
-        """
-        code_block_ranges = []
-
-        # Match code block as well as `code` and `sourcecode` aliases
-        code_block_pattern = re.compile(
-            r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
-        )
-
-        for match in code_block_pattern.finditer(content):
-            start_pos = match.start()
-            indent = match.group(1)
-            indent_len = len(indent)
-
-            # Find the end of the code block by looking for the next line
-            # that is not indented more than the directive
-            block_start = match.end()
-            pos = block_start
-
-            # Skip any blank lines immediately after the directive
-            while pos < len(content) and content[pos] in "\n":
-                pos += 1
-
-            # Find where the code block ends
-            lines = content[pos:].split("\n")
-            block_end = pos
-            for line in lines:
-                if line.strip():  # Non-empty line
-                    # Check indentation level
-                    line_indent = len(line) - len(line.lstrip())
-                    if line_indent <= indent_len:
-                        # The block ends when we find a line that is indented
-                        # less than the directive itself
-                        break
-                block_end += len(line) + 1  # +1 for the newline
-
-            code_block_ranges.append((start_pos, block_end))
-
-        return code_block_ranges
-
-    def _is_in_code_block(
-        self, match_start: int, code_block_ranges: List[Tuple[int, int]]
-    ) -> bool:
-        """Check if a match position is within a code block.
-
-        Args:
-            match_start: The starting position of the match
-            code_block_ranges: List of (start, end) tuples for code blocks
-
-        Returns:
-            True if the match is within a code block, False otherwise
-        """
-        for block_start, block_end in code_block_ranges:
-            if block_start <= match_start < block_end:
-                return True
-        return False
-
     def _process_includes(self, content: str, source_path: Path) -> str:
         """Process include directives in content.
 
@@ -359,18 +286,11 @@ class DocumentProcessor:
         Returns:
             Processed content with include directives replaced with included content
         """
-        code_block_ranges = self._get_code_block_ranges(content)
-
         # Find all include directives using regex
         include_pattern = build_directive_pattern(["include"])
 
         # Function to replace each include with content
         def replace_include(match):
-            # Check if this include is within a code block
-            if self._is_in_code_block(match.start(), code_block_ranges):
-                # This include is inside a code block, don't process it
-                return match.group(0)
-
             include_path = match.group(3)
             directive_part = match.group(
                 1

tests/test_llms_txt.py

@@ -198,31 +198,6 @@ def test_process_includes(tmp_path):
     assert processed_content == expected_content
 
 
-def test_process_includes_in_code_block(tmp_path):
-    """Test that an `include` within a `code-block` is not processed."""
-    # Create a processor
-    config = {"llms_txt_directives": []}
-    processor = DocumentProcessor(config)
-
-    # Create a source file that uses include syntax within a `code-block`
-    source_content = (
-        "Normal paragraph.\n\n"
-        ".. code-block:: rst\n\n"
-        "   .. include:: foo.txt\n\n"
-        "Another normal paragraph."
-    )
-    source_file = tmp_path / "source.txt"
-    with open(source_file, "w", encoding="utf-8") as f:
-        f.write(source_content)
-
-    # Run the include directive processor
-    processed_content = processor._process_includes(source_content, source_file)
-
-    # Check that the include directive was not processed
-    expected_content = source_content
-    assert processed_content == expected_content
-
-
 def test_process_includes_with_relative_paths(tmp_path):
     """Test that include directives with relative paths are processed correctly."""
     # Create a processor