Don't process includes within code blocks (#58)

Updated footnote reference for sphinx-llm to clarify usage.

.readthedocs.yml

@@ -9,6 +9,10 @@ build:
     - pip install -r docs/requirements.txt
     - pip install -e .
     - cmake --workflow --preset documentation-workflow
+    # Generate llms.txt variants for demo purposes
+    - python docs/generate_llms_variants.py build/html
     # Copy built documentation to Read the Docs output directory
     - mkdir -p $READTHEDOCS_OUTPUT/html
-    - cp -r build/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/rst/* $READTHEDOCS_OUTPUT/html/

CHANGELOG.rst

@@ -1,6 +1,11 @@
 Changelog
 =========
 
+0.7.1
+-----
+
+- Don't process includes within code blocks
+
 0.7.0
 -----
 

CMakeLists.txt

@@ -1,8 +1,15 @@
 cmake_minimum_required(VERSION 3.15)
-project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
+project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
 
-# Add CMake module path
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+# Fetch Sphinx CMake modules
+include(FetchContent)
+FetchContent_Declare(
+  sphinx_cmake_modules
+  GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
+  GIT_TAG        main
+)
+FetchContent_MakeAvailable(sphinx_cmake_modules)
+list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
 
 # Add documentation
 add_subdirectory(docs)

CMakePresets.json

@@ -22,10 +22,16 @@
       "targets": ["markdown"]
     },
     {
-      "name": "docs-parallel",
-      "displayName": "Build HTML and Markdown in parallel",
+      "name": "rst",
+      "displayName": "Build reStructuredText Documentation",
       "configurePreset": "documentation",
-      "targets": ["html", "markdown"]
+      "targets": ["rst"]
+    },
+    {
+      "name": "docs-parallel",
+      "displayName": "Build all output formats in parallel",
+      "configurePreset": "documentation",
+      "targets": ["html", "markdown", "rst"]
     }
   ],
   "workflowPresets": [

cmake/SetupSphinx.cmake

@@ -1,27 +0,0 @@
-# SetupSphinx.cmake - Find Sphinx for CI environments
-
-# Function to find Sphinx in the system
-function(setup_sphinx_environment)
-    # Find sphinx-build executable in system
-    find_program(SPHINX_EXECUTABLE
-        NAMES sphinx-build
-        DOC "Sphinx documentation generator"
-    )
-
-    if(NOT SPHINX_EXECUTABLE)
-        message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
-    endif()
-
-    # Get Sphinx version
-    execute_process(
-        COMMAND ${SPHINX_EXECUTABLE} --version
-        OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
-        ERROR_VARIABLE SPHINX_VERSION_OUTPUT
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
-    message(STATUS "${SPHINX_VERSION_OUTPUT}")
-
-    # Export to parent scope
-    set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
-endfunction()

docs/CMakeLists.txt

@@ -1,24 +1,7 @@
-# Find Sphinx
-include(SetupSphinx)
+include(SphinxUtils)
+
 setup_sphinx_environment()
 
-set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
-set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
-
-# Build HTML documentation
-add_custom_target(html
-    COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
-    COMMENT "Building HTML documentation"
-    VERBATIM
-)
-
-# Build markdown documentation
-add_custom_target(markdown
-    COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
-    COMMENT "Building Markdown documentation"
-    VERBATIM
-)
-
-# Build both formats
-add_custom_target(docs ALL)
-add_dependencies(docs html markdown)
+add_sphinx_builder(html)
+add_sphinx_builder(markdown)
+add_sphinx_builder(rst)

docs/generate_llms_variants.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Generate variant llms.txt files for demo purposes.
+
+Takes the generated llms.txt (with _sources links) and creates:
+- llms.txt - default with _sources links (unchanged)
+- llms.md.txt - .html.md links
+- llms.rst.txt - .rst links
+"""
+
+import re
+import sys
+from pathlib import Path
+
+
+def get_base_url() -> str:
+    """Import base_url from conf.py."""
+    sys.path.insert(0, str(Path(__file__).parent / "source"))
+    from conf import html_baseurl  # noqa: E402
+
+    return html_baseurl
+
+
+def generate_variants(build_dir: Path) -> None:
+    """Generate llms.txt variants from the original file."""
+    original = build_dir / "llms.txt"
+
+    if not original.exists():
+        print(f"Error: {original} not found")
+        sys.exit(1)
+
+    content = original.read_text()
+    base_url = get_base_url()
+
+    # Pattern to match links like: https://.../_sources/{docname}.rst.txt
+    link_pattern = re.compile(
+        rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
+    )
+
+    # Generate .html.md variant
+    md_content = link_pattern.sub(r"\1\2.html.md", content)
+    (build_dir / "llms.md.txt").write_text(md_content)
+    print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
+
+    # Generate .rst variant
+    rst_content = link_pattern.sub(r"\1\2.rst", content)
+    (build_dir / "llms.rst.txt").write_text(rst_content)
+    print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
+
+    print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        build_dir = Path(sys.argv[1])
+    else:
+        build_dir = Path("build/html")
+
+    generate_variants(build_dir)

docs/requirements.txt

@@ -2,6 +2,9 @@ furo
 esbonio
 sphinx-contributors
 sphinx
+sphinx-design
 sphinx-llms-txt
+sphinx-inline-tabs
 sphinxext-opengraph
 sphinx-markdown-builder
+sphinxcontrib-restbuilder

docs/source/advanced-configuration.rst

@@ -292,6 +292,147 @@ Your URI template can use the following variables:
 - ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
 - ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
 
+.. tip::
+   Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
+   See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
+   Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
+
+.. _cmake_workflow:
+
+CMake Workflow
+^^^^^^^^^^^^^^
+
+This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
+This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
+
+Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
+Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
+
+Key Files
+~~~~~~~~~
+
+These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
+
+.. code-block:: text
+
+   .
+   ├── .readthedocs.yml
+   ├── CMakeLists.txt
+   ├── CMakePresets.json
+   └── docs/
+       └── CMakeLists.txt
+
+Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
+
+.. dropdown:: .readthedocs.yml
+   :chevron: down-up
+
+   A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
+
+   .. literalinclude:: ../../.readthedocs.yml
+      :language: yaml
+      :lines: 1-9,11,14-
+      :linenos:
+      :emphasize-lines: 9, 14-15
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **9**
+        - Update the path if your requirements file is in a different location
+      * - **13-14**
+        - Modify the copy commands for the output formats you deploy
+
+.. dropdown:: CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
+
+   .. literalinclude:: ../../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 9, 15
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **9**
+        - Update the ``GIT_TAG`` to use a different version or commit hash
+      * - **15**
+        - Change if your docs subdirectory has a different location
+
+.. dropdown:: docs/CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
+
+   .. literalinclude:: ../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 5-7
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **5-7**
+        - Add or remove calls based on which output formats you need
+
+
+.. dropdown:: CMakePresets.json
+   :chevron: down-up
+
+   Defines presets for configuring and building documentation:
+
+   - **Configure Presets:** Sets up the build directory.
+   - **Build Presets:** Defines build formats individually and all in parallel.
+   - **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
+
+   .. literalinclude:: ../../CMakePresets.json
+      :language: json
+      :linenos:
+      :emphasize-lines: 18-23, 24-29, 34
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **18-23**
+        - Remove this preset to disable Markdown documentation builds
+      * - **24-29**
+        - Remove this preset to disable reStructuredText documentation builds
+      * - **34**
+        - Modify the targets list to build only the output formats you need in parallel
+
+Usage
+~~~~~
+
+To build documentation locally using CMake:
+
+.. code-block:: console
+
+   # Run the full workflow (configure + build all formats)
+   cmake --workflow --preset documentation-workflow
+
+   # Or configure and build separately
+   cmake --preset documentation
+   cmake --build --preset html        # Build HTML only
+   cmake --build --preset docs-parallel  # Build all formats
+
 .. _integration_examples:
 
 Integration Examples

docs/source/conf.py

@@ -15,13 +15,17 @@ import subprocess
 project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
-llms_txt_uri_template = "{base_url}{docname}.md"
+
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
 A Sphinx extension that generates a summary llms.txt file,written in Markdown,
 and a single combined documentation llms-full.txt file, written in reStructuredText.
 """
 
+# This doesn't seem to be supported
+# rst_file_suffix = ".html.rst"
+markdown_file_suffix = ".html.md"
+
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
     GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -50,6 +54,10 @@ extensions = [
     "sphinx.ext.intersphinx",
     "sphinx_contributors",
     "sphinx_llms_txt",
+    "sphinxcontrib.restbuilder",
+    "sphinx_inline_tabs",
+    "sphinx.ext.extlinks",
+    "sphinx_design",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/source/getting-started.rst

@@ -4,17 +4,19 @@ Getting Started
 Installation
 ------------
 
-Directly install via ``pip`` by using:
+Directly install by using:
 
-.. code-block:: bash
+.. tab:: via pip
 
-    pip install sphinx-llms-txt
+   .. code-block:: bash
 
-Or with ``conda`` via ``conda-forge``:
+      pip install sphinx-llms-txt
 
-.. code::
+.. tab:: via conda:
 
-   conda install -c conda-forge sphinx-llms-txt
+   .. code-block:: bash
+
+      conda install -c conda-forge sphinx-llms-txt
 
 Usage
 -----
@@ -32,7 +34,59 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
     sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
     sphinx-llms-txt: created /path/to/_build/html/llms.txt
 
+.. _choosing-output-format:
 
-.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades.
+Choosing an Output Format
+-------------------------
+
+By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
+For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
+
+.. list-table:: Output Format Comparison
+   :header-rows: 1
+   :widths: 18 27 27 27
+
+   * -
+     - Default
+     - Markdown
+     - reStructuredText
+   * - **Setup**
+     - No config
+     - CMake [#sphinxllm]_
+     - CMake
+   * - **Builder**
+     - Native [#native]_
+     - `sphinx-markdown-builder`_
+     - `sphinxcontrib-restbuilder`_
+   * - **Format**
+     - Raw reStructuredText source
+     - Rendered Markdown [#rendered]_
+     - Rendered reStructuredText [#rendered]_
+   * - **LLM Readability**
+     - Good - preserves structure for simple syntax
+     - Excellent - native LLM format
+     - Good - Can provide more structured content
+   * - **Key Advantage**
+     - Zero setup required
+     - More compact (less input tokens)
+     - Can preserve Sphinx semantics
+   * - **Key Disadvantage**
+     - Raw directives won't be parsed [#autodoc]_
+     - Loses structure from complex directives
+     - Can lose structure from complex directives
+   * - **llms-full.txt support**
+     - Supported with above caveats
+     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
+     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
+
+.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
+.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
+
+.. rubric:: Footnotes
+
+.. [#sphinxllm] See `sphinx-llm <https://github.com/jacobtomlinson/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
+.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
+.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
+.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
+.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
 
-See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.

docs/source/index.rst

@@ -8,21 +8,23 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
 Demo
 ----
 
-You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example.
+This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
+
+Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
 
 Highlights
 ----------
 
-1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build
-2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content
-3. **Path Resolution**: Transforms relative paths in directives to full paths
-4. **Output Generation**: Creates two optional files:
+**Zero Configuration**
+   Add the extension to your ``conf.py`` and you're done.
+   The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
 
-   - ``llms.txt``: A concise summary of your documentation, in Markdown
-   - ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText
+**Intelligent Content Processing**
+   Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
 
-5. **Content Filtering**: Allows you to exclude specific pages or sections
-6. **Source Code**: Allows you to include specific source code files
+**Customizable When Needed**
+   Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
+   See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
 
 .. toctree::
    :maxdepth: 2
@@ -36,6 +38,8 @@ Highlights
 
 .. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
 .. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
+.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
+.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
 .. _Sphinx: http://sphinx-doc.org/
 
 .. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg

sphinx_llms_txt/__init__.py

@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.7.0"
+__version__ = "0.7.1"
 
 # Export classes needed by tests
 __all__ = [

sphinx_llms_txt/processor.py

@@ -128,8 +128,11 @@ class DocumentProcessor:
         Returns:
             Processed content with directive paths properly resolved
         """
+        # Get code block ranges to skip directives inside them
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Get the configured path directives to process
-        default_path_directives = ["image", "figure"]
+        default_path_directives = ["image", "figure", "literalinclude"]
         custom_path_directives = self.config.get("llms_txt_directives")
         path_directives = set(default_path_directives + custom_path_directives)
 
@@ -143,6 +146,11 @@ class DocumentProcessor:
         is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
 
         def replace_directive_path(match, base_url=base_url, is_test=is_test):
+            # Check if this directive is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This directive is inside a code block, don't process it
+                return match.group(0)
+
             prefix = match.group(1)  # The entire directive prefix including whitespace
             path = match.group(3).strip()  # The path argument
 
@@ -276,6 +284,71 @@ class DocumentProcessor:
 
         return possible_paths
 
+    def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
+        """Find all code block ranges in the content.
+
+        Args:
+            content: The source content to analyze
+
+        Returns:
+            List of (start, end) tuples representing code block character
+            ranges
+        """
+        code_block_ranges = []
+
+        # Match code block as well as `code` and `sourcecode` aliases
+        code_block_pattern = re.compile(
+            r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
+        )
+
+        for match in code_block_pattern.finditer(content):
+            start_pos = match.start()
+            indent = match.group(1)
+            indent_len = len(indent)
+
+            # Find the end of the code block by looking for the next line
+            # that is not indented more than the directive
+            block_start = match.end()
+            pos = block_start
+
+            # Skip any blank lines immediately after the directive
+            while pos < len(content) and content[pos] in "\n":
+                pos += 1
+
+            # Find where the code block ends
+            lines = content[pos:].split("\n")
+            block_end = pos
+            for line in lines:
+                if line.strip():  # Non-empty line
+                    # Check indentation level
+                    line_indent = len(line) - len(line.lstrip())
+                    if line_indent <= indent_len:
+                        # The block ends when we find a line that is indented
+                        # less than the directive itself
+                        break
+                block_end += len(line) + 1  # +1 for the newline
+
+            code_block_ranges.append((start_pos, block_end))
+
+        return code_block_ranges
+
+    def _is_in_code_block(
+        self, match_start: int, code_block_ranges: List[Tuple[int, int]]
+    ) -> bool:
+        """Check if a match position is within a code block.
+
+        Args:
+            match_start: The starting position of the match
+            code_block_ranges: List of (start, end) tuples for code blocks
+
+        Returns:
+            True if the match is within a code block, False otherwise
+        """
+        for block_start, block_end in code_block_ranges:
+            if block_start <= match_start < block_end:
+                return True
+        return False
+
     def _process_includes(self, content: str, source_path: Path) -> str:
         """Process include directives in content.
 
@@ -286,11 +359,18 @@ class DocumentProcessor:
         Returns:
             Processed content with include directives replaced with included content
         """
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Find all include directives using regex
         include_pattern = build_directive_pattern(["include"])
 
         # Function to replace each include with content
         def replace_include(match):
+            # Check if this include is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This include is inside a code block, don't process it
+                return match.group(0)
+
             include_path = match.group(3)
             directive_part = match.group(
                 1

tests/test_llms_txt.py

@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
     assert processed_content == expected_content
 
 
+def test_process_includes_in_code_block(tmp_path):
+    """Test that an `include` within a `code-block` is not processed."""
+    # Create a processor
+    config = {"llms_txt_directives": []}
+    processor = DocumentProcessor(config)
+
+    # Create a source file that uses include syntax within a `code-block`
+    source_content = (
+        "Normal paragraph.\n\n"
+        ".. code-block:: rst\n\n"
+        "   .. include:: foo.txt\n\n"
+        "Another normal paragraph."
+    )
+    source_file = tmp_path / "source.txt"
+    with open(source_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Run the include directive processor
+    processed_content = processor._process_includes(source_content, source_file)
+
+    # Check that the include directive was not processed
+    expected_content = source_content
+    assert processed_content == expected_content
+
+
 def test_process_includes_with_relative_paths(tmp_path):
     """Test that include directives with relative paths are processed correctly."""
     # Create a processor