Apply black formatting

Fixes #57

.readthedocs.yml

@@ -11,4 +11,6 @@ build:
     - cmake --workflow --preset documentation-workflow
     # Copy built documentation to Read the Docs output directory
     - mkdir -p $READTHEDOCS_OUTPUT/html
-    - cp -r build/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/rst/* $READTHEDOCS_OUTPUT/html/

CMakePresets.json

@@ -21,11 +21,17 @@
       "configurePreset": "documentation",
       "targets": ["markdown"]
     },
+    {
+      "name": "rst",
+      "displayName": "Build reStructuredText Documentation",
+      "configurePreset": "documentation",
+      "targets": ["rst"]
+    },
     {
       "name": "docs-parallel",
       "displayName": "Build HTML and Markdown in parallel",
       "configurePreset": "documentation",
-      "targets": ["html", "markdown"]
+      "targets": ["html", "markdown", "rst"]
     }
   ],
   "workflowPresets": [

cmake/SphinxUtils.cmake

@@ -1,4 +1,7 @@
-# SetupSphinx.cmake - Find Sphinx for CI environments
+# Sphinx related utilities
+
+set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
+set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
 
 # Function to find Sphinx in the system
 function(setup_sphinx_environment)
@@ -12,16 +15,14 @@ function(setup_sphinx_environment)
         message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
     endif()
 
-    # Get Sphinx version
-    execute_process(
-        COMMAND ${SPHINX_EXECUTABLE} --version
-        OUTPUT_VARIABLE SPHINX_VERSION_OUTPUT
-        ERROR_VARIABLE SPHINX_VERSION_OUTPUT
-        OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    message(STATUS "Found Sphinx: ${SPHINX_EXECUTABLE}")
-    message(STATUS "${SPHINX_VERSION_OUTPUT}")
-
     # Export to parent scope
     set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
 endfunction()
+
+# Function to add a Sphinx builder target
+function(add_sphinx_builder builder_name)
+    add_custom_target(${builder_name}
+        COMMAND ${SPHINX_EXECUTABLE} -b ${builder_name} ${SPHINX_SOURCE} ${SPHINX_BUILD}/${builder_name}
+        VERBATIM
+    )
+endfunction()

docs/CMakeLists.txt

@@ -1,24 +1,7 @@
-# Find Sphinx
+include(SphinxUtils)
-include(SetupSphinx)
+
 setup_sphinx_environment()
 
-set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
+add_sphinx_builder(html)
-set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
+add_sphinx_builder(markdown)
-
+add_sphinx_builder(rst)
-# Build HTML documentation
-add_custom_target(html
-    COMMAND ${SPHINX_EXECUTABLE} -b html ${SPHINX_SOURCE} ${SPHINX_BUILD}
-    COMMENT "Building HTML documentation"
-    VERBATIM
-)
-
-# Build markdown documentation
-add_custom_target(markdown
-    COMMAND ${SPHINX_EXECUTABLE} -b markdown ${SPHINX_SOURCE} ${SPHINX_BUILD}
-    COMMENT "Building Markdown documentation"
-    VERBATIM
-)
-
-# Build both formats
-add_custom_target(docs ALL)
-add_dependencies(docs html markdown)

docs/requirements.txt

@@ -3,5 +3,7 @@ esbonio
 sphinx-contributors
 sphinx
 sphinx-llms-txt
+sphinx-inline-tabs
 sphinxext-opengraph
 sphinx-markdown-builder
+sphinxcontrib-restbuilder

docs/source/advanced-configuration.rst

@@ -292,6 +292,68 @@ Your URI template can use the following variables:
 - ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
 - ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
 
+.. tip::
+   Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
+   See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
+   Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
+
+.. _cmake_workflow:
+
+CMake Workflow
+^^^^^^^^^^^^^^
+
+This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
+Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
+This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
+
+Key Files
+~~~~~~~~~
+
+.. code-block:: text
+
+   .
+   ├── .readthedocs.yml
+   ├── CMakeLists.txt
+   ├── CMakePresets.json
+   ├── cmake/
+   │   └── SphinxUtils.cmake
+   └── docs/
+       └── CMakeLists.txt
+
+:ghfile:`.readthedocs.yml`
+   A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
+
+:ghfile:`CMakeLists.txt`
+   A CMake config file that sets up the project and includes the ``cmake/`` module path.
+
+:ghfile:`docs/CMakeLists.txt`
+   A CMake config file that includes the Sphinx utilities and defines the documentation-specific build targets.
+
+:ghfile:`cmake/SphinxUtils.cmake`
+   A CMake module that provides Sphinx related utilities.
+
+:ghfile:`CMakePresets.json`
+   Defines presets for configuring and building documentation:
+
+   - **Configure Presets:** Sets up the build directory.
+   - **Build Presets:** Defines Build formats individually and all in parallel.
+   - **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
+
+Usage
+~~~~~
+
+To build documentation locally using CMake:
+
+.. code-block:: console
+
+   # Run the full workflow (configure + build all formats)
+   cmake --workflow --preset documentation-workflow
+
+   # Or configure and build separately
+   cmake --preset documentation
+   cmake --build --preset html        # Build HTML only
+   cmake --build --preset docs-parallel  # Build all formats
+
 .. _integration_examples:
 
 Integration Examples

docs/source/conf.py

@@ -15,6 +15,7 @@ import subprocess
 project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
+
 llms_txt_uri_template = "{base_url}{docname}.md"
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
@@ -22,6 +23,13 @@ A Sphinx extension that generates a summary llms.txt file,written in Markdown,
 and a single combined documentation llms-full.txt file, written in reStructuredText.
 """
 
+# This doesn't seem to be supported
+# rst_file_suffix = ".html.rst"
+
+extlinks = {
+    "ghfile": ("https://github.com/jdillard/sphinx-llms-txt/blob/main/%s", "%s")
+}
+
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
     GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -50,6 +58,9 @@ extensions = [
     "sphinx.ext.intersphinx",
     "sphinx_contributors",
     "sphinx_llms_txt",
+    "sphinxcontrib.restbuilder",
+    "sphinx_inline_tabs",
+    "sphinx.ext.extlinks",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/source/getting-started.rst

@@ -4,15 +4,17 @@ Getting Started
 Installation
 ------------
 
-Directly install via ``pip`` by using:
+Directly install by using:
+
+.. tab:: via pip
 
    .. code-block:: bash
 
       pip install sphinx-llms-txt
 
-Or with ``conda`` via ``conda-forge``:
+.. tab:: via conda:
 
-.. code::
+   .. code-block:: bash
 
       conda install -c conda-forge sphinx-llms-txt
 
@@ -32,7 +34,45 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
     sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
     sphinx-llms-txt: created /path/to/_build/html/llms.txt
 
+Choosing an Output Format
+-------------------------
 
-.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades.
+By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files in :confval:`_sources/ <sphinx:html_copy_source>`.
+For optimal LLM support, you can use `sphinx-markdown-builder`_ and/or `sphinxcontrib-restbuilder`_, set up in parallel builds using :ref:`CMake <cmake_workflow>`.
+
+.. list-table:: Output Format Comparison
+   :header-rows: 1
+   :widths: 18 27 27 27
+
+   * -
+     - Default (no config)
+     - Markdown (CMake)
+     - RST (CMake)
+   * - **Format**
+     - Raw RST source
+     - Rendered Markdown
+     - Rendered RST
+   * - **LLM Readability**
+     - Good - preserves structure for simple syntax
+     - Excellent - native LLM format
+     - Good - Can provide more structured content
+   * - **Key Advantage**
+     - Zero setup required
+     - More compact (less input tokens)
+     - Can preserve Sphinx semantics
+   * - **Key Disadvantage**
+     - Raw directives (e.g., autodoc) won't be parsed
+     - Loses structure from complex directives
+     - Can lose structure from complex directives
+   * - **llms-full.txt support**
+     - Suported with above caveats
+     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__
+     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__
+
+
+See :ref:`cmake_workflow` for an example of building HTML, Markdown, and RST in parallel.
+Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
+
+.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
+.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
 
-See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.

docs/source/index.rst

@@ -13,16 +13,16 @@ You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a si
 Highlights
 ----------
 
-1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build
+**Zero Configuration**
-2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content
+   Add the extension to your ``conf.py`` and you're done.
-3. **Path Resolution**: Transforms relative paths in directives to full paths
+   The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
-4. **Output Generation**: Creates two optional files:
 
-   - ``llms.txt``: A concise summary of your documentation, in Markdown
+**Intelligent Content Processing**
-   - ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText
+   Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
 
-5. **Content Filtering**: Allows you to exclude specific pages or sections
+**Customizable When Needed**
-6. **Source Code**: Allows you to include specific source code files
+   Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
+   See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
 
 .. toctree::
    :maxdepth: 2

sphinx_llms_txt/processor.py

@@ -128,8 +128,11 @@ class DocumentProcessor:
         Returns:
             Processed content with directive paths properly resolved
         """
+        # Get code block ranges to skip directives inside them
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Get the configured path directives to process
-        default_path_directives = ["image", "figure"]
+        default_path_directives = ["image", "figure", "literalinclude"]
         custom_path_directives = self.config.get("llms_txt_directives")
         path_directives = set(default_path_directives + custom_path_directives)
 
@@ -143,6 +146,11 @@ class DocumentProcessor:
         is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
 
         def replace_directive_path(match, base_url=base_url, is_test=is_test):
+            # Check if this directive is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This directive is inside a code block, don't process it
+                return match.group(0)
+
             prefix = match.group(1)  # The entire directive prefix including whitespace
             path = match.group(3).strip()  # The path argument
 
@@ -276,6 +284,71 @@ class DocumentProcessor:
 
         return possible_paths
 
+    def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
+        """Find all code block ranges in the content.
+
+        Args:
+            content: The source content to analyze
+
+        Returns:
+            List of (start, end) tuples representing code block character
+            ranges
+        """
+        code_block_ranges = []
+
+        # Match code block as well as `code` and `sourcecode` aliases
+        code_block_pattern = re.compile(
+            r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
+        )
+
+        for match in code_block_pattern.finditer(content):
+            start_pos = match.start()
+            indent = match.group(1)
+            indent_len = len(indent)
+
+            # Find the end of the code block by looking for the next line
+            # that is not indented more than the directive
+            block_start = match.end()
+            pos = block_start
+
+            # Skip any blank lines immediately after the directive
+            while pos < len(content) and content[pos] in "\n":
+                pos += 1
+
+            # Find where the code block ends
+            lines = content[pos:].split("\n")
+            block_end = pos
+            for line in lines:
+                if line.strip():  # Non-empty line
+                    # Check indentation level
+                    line_indent = len(line) - len(line.lstrip())
+                    if line_indent <= indent_len:
+                        # The block ends when we find a line that is indented
+                        # less than the directive itself
+                        break
+                block_end += len(line) + 1  # +1 for the newline
+
+            code_block_ranges.append((start_pos, block_end))
+
+        return code_block_ranges
+
+    def _is_in_code_block(
+        self, match_start: int, code_block_ranges: List[Tuple[int, int]]
+    ) -> bool:
+        """Check if a match position is within a code block.
+
+        Args:
+            match_start: The starting position of the match
+            code_block_ranges: List of (start, end) tuples for code blocks
+
+        Returns:
+            True if the match is within a code block, False otherwise
+        """
+        for block_start, block_end in code_block_ranges:
+            if block_start <= match_start < block_end:
+                return True
+        return False
+
     def _process_includes(self, content: str, source_path: Path) -> str:
         """Process include directives in content.
 
@@ -286,11 +359,18 @@ class DocumentProcessor:
         Returns:
             Processed content with include directives replaced with included content
         """
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Find all include directives using regex
         include_pattern = build_directive_pattern(["include"])
 
         # Function to replace each include with content
         def replace_include(match):
+            # Check if this include is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This include is inside a code block, don't process it
+                return match.group(0)
+
             include_path = match.group(3)
             directive_part = match.group(
                 1

tests/test_llms_txt.py

@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
     assert processed_content == expected_content
 
 
+def test_process_includes_in_code_block(tmp_path):
+    """Test that an `include` within a `code-block` is not processed."""
+    # Create a processor
+    config = {"llms_txt_directives": []}
+    processor = DocumentProcessor(config)
+
+    # Create a source file that uses include syntax within a `code-block`
+    source_content = (
+        "Normal paragraph.\n\n"
+        ".. code-block:: rst\n\n"
+        "   .. include:: foo.txt\n\n"
+        "Another normal paragraph."
+    )
+    source_file = tmp_path / "source.txt"
+    with open(source_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Run the include directive processor
+    processed_content = processor._process_includes(source_content, source_file)
+
+    # Check that the include directive was not processed
+    expected_content = source_content
+    assert processed_content == expected_content
+
+
 def test_process_includes_with_relative_paths(tmp_path):
     """Test that include directives with relative paths are processed correctly."""
     # Create a processor