Fix encoding of Unicode characters

Bumps the all-github-actions group with 1 update: [actions/checkout](https://github.com/actions/checkout).


Updates `actions/checkout` from 5 to 6
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
   pre-commit:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Set up Python 3.10
         uses: actions/setup-python@v6
         with:
@@ -23,7 +23,7 @@ jobs:
         python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6

.readthedocs.yml

@@ -1,15 +1,18 @@
 version: 2
 
 build:
-  os: "ubuntu-20.04"
+  os: ubuntu-24.04
   tools:
-    python: "3.10"
-
-sphinx:
-  configuration: docs/source/conf.py
-
-python:
-  install:
-    - requirements: docs/requirements.txt
-    - method: pip
-      path: .
+    python: "3.13"
+  commands:
+    - pip install cmake
+    - pip install -r docs/requirements.txt
+    - pip install -e .
+    - cmake --workflow --preset documentation-workflow
+    # Generate llms.txt variants for demo purposes
+    - python docs/generate_llms_variants.py build/html
+    # Copy built documentation to Read the Docs output directory
+    - mkdir -p $READTHEDOCS_OUTPUT/html
+    - cp -r build/html/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/markdown/* $READTHEDOCS_OUTPUT/html/
+    - cp -r build/rst/* $READTHEDOCS_OUTPUT/html/

CHANGELOG.rst

@@ -1,6 +1,23 @@
 Changelog
 =========
 
+0.7.2
+-----
+
+- Fix encoding of Unicode characters (smart quotes, em dashes, etc.) in llms.txt output
+  `#65 <https://github.com/jdillard/sphinx-llms-txt/issues/65>`_
+
+0.7.1
+-----
+
+- Don't process includes within code blocks
+
+0.7.0
+-----
+
+- Add :confval:`llms_txt_uri_template` configuration option to control the link behavior in :confval:`llms_txt_filename`.
+  `#48 <https://github.com/jdillard/sphinx-llms-txt/pull/48>`_
+
 0.6.0
 -----
 

CMakeLists.txt

@@ -0,0 +1,15 @@
+cmake_minimum_required(VERSION 3.15)
+project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
+
+# Fetch Sphinx CMake modules
+include(FetchContent)
+FetchContent_Declare(
+  sphinx_cmake_modules
+  GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
+  GIT_TAG        main
+)
+FetchContent_MakeAvailable(sphinx_cmake_modules)
+list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
+
+# Add documentation
+add_subdirectory(docs)

CMakePresets.json

@@ -0,0 +1,53 @@
+{
+  "version": 6,
+  "configurePresets": [
+    {
+      "name": "documentation",
+      "displayName": "Documentation Build",
+      "description": "Configure project with documentation environment setup",
+      "binaryDir": "${sourceDir}/build"
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "html",
+      "displayName": "Build HTML Documentation",
+      "configurePreset": "documentation",
+      "targets": ["html"]
+    },
+    {
+      "name": "markdown",
+      "displayName": "Build Markdown Documentation",
+      "configurePreset": "documentation",
+      "targets": ["markdown"]
+    },
+    {
+      "name": "rst",
+      "displayName": "Build reStructuredText Documentation",
+      "configurePreset": "documentation",
+      "targets": ["rst"]
+    },
+    {
+      "name": "docs-parallel",
+      "displayName": "Build all output formats in parallel",
+      "configurePreset": "documentation",
+      "targets": ["html", "markdown", "rst"]
+    }
+  ],
+  "workflowPresets": [
+    {
+      "name": "documentation-workflow",
+      "displayName": "Documentation Build Workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "documentation"
+        },
+        {
+          "type": "build",
+          "name": "docs-parallel"
+        }
+      ]
+    }
+  ]
+}

docs/CMakeLists.txt

@@ -0,0 +1,7 @@
+include(SphinxUtils)
+
+setup_sphinx_environment()
+
+add_sphinx_builder(html)
+add_sphinx_builder(markdown)
+add_sphinx_builder(rst)

docs/generate_llms_variants.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Generate variant llms.txt files for demo purposes.
+
+Takes the generated llms.txt (with _sources links) and creates:
+- llms.txt - default with _sources links (unchanged)
+- llms.md.txt - .html.md links
+- llms.rst.txt - .rst links
+"""
+
+import re
+import sys
+from pathlib import Path
+
+
+def get_base_url() -> str:
+    """Import base_url from conf.py."""
+    sys.path.insert(0, str(Path(__file__).parent / "source"))
+    from conf import html_baseurl  # noqa: E402
+
+    return html_baseurl
+
+
+def generate_variants(build_dir: Path) -> None:
+    """Generate llms.txt variants from the original file."""
+    original = build_dir / "llms.txt"
+
+    if not original.exists():
+        print(f"Error: {original} not found")
+        sys.exit(1)
+
+    content = original.read_text()
+    base_url = get_base_url()
+
+    # Pattern to match links like: https://.../_sources/{docname}.rst.txt
+    link_pattern = re.compile(
+        rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
+    )
+
+    # Generate .html.md variant
+    md_content = link_pattern.sub(r"\1\2.html.md", content)
+    (build_dir / "llms.md.txt").write_text(md_content)
+    print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
+
+    # Generate .rst variant
+    rst_content = link_pattern.sub(r"\1\2.rst", content)
+    (build_dir / "llms.rst.txt").write_text(rst_content)
+    print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
+
+    print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        build_dir = Path(sys.argv[1])
+    else:
+        build_dir = Path("build/html")
+
+    generate_variants(build_dir)

docs/requirements.txt

@@ -2,5 +2,9 @@ furo
 esbonio
 sphinx-contributors
 sphinx
+sphinx-design
 sphinx-llms-txt
+sphinx-inline-tabs
 sphinxext-opengraph
+sphinx-markdown-builder
+sphinxcontrib-restbuilder

docs/source/advanced-configuration.rst

@@ -261,6 +261,178 @@ If you want to include absolute URLs for resources in your documentation, you ca
 
 When this option is set, all resolved paths in directives will be prefixed with this URL, creating absolute paths in the generated files.
 
+.. _customizing_uri_links:
+
+Customizing URI Links in llms.txt
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default, the ``llms.txt`` file links to source files in the ``_sources`` directory when available, falling back to HTML pages when sources aren't available.
+You can customize this behavior using URI templates with :confval:`llms_txt_uri_template`:
+
+.. code-block:: python
+
+   # Default: Link to source files, if _sources exists
+   llms_txt_uri_template = "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+
+   # Default: Link to HTML pages instead, if _sources doesn't exist
+   llms_txt_uri_template = "{base_url}{docname}.html"
+
+   # Manual: Link to a custom markdown build
+   llms_txt_uri_template = "{base_url}{docname}.md"
+
+.. _available_template_variables:
+
+Available Template Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Your URI template can use the following variables:
+
+- ``{base_url}`` - The base URL from ``html_baseurl`` configuration (includes trailing slash)
+- ``{docname}`` - The document name (e.g., ``index``, ``guide/intro``)
+- ``{suffix}`` - The source file suffix (e.g., ``.rst``, ``.md``) - may be empty if no source file exists
+- ``{sourcelink_suffix}`` - The suffix from ``html_sourcelink_suffix`` configuration (e.g., ``.txt``)
+
+.. tip::
+   Instead of using the default of linking to ``_sources``, you can generate Markdown and/or reStructuredText files from your documentation and link to those in ``llms.txt``.
+   See :ref:`cmake_workflow` for an example of building both HTML and Markdown and/or reStructuredText in parallel.
+   Note that ``_sources`` is still needed for ``llms-full.txt`` at this time.
+
+.. _cmake_workflow:
+
+CMake Workflow
+^^^^^^^^^^^^^^
+
+This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
+This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
+
+Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
+Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
+
+Key Files
+~~~~~~~~~
+
+These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
+
+.. code-block:: text
+
+   .
+   ├── .readthedocs.yml
+   ├── CMakeLists.txt
+   ├── CMakePresets.json
+   └── docs/
+       └── CMakeLists.txt
+
+Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
+
+.. dropdown:: .readthedocs.yml
+   :chevron: down-up
+
+   A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
+
+   .. literalinclude:: ../../.readthedocs.yml
+      :language: yaml
+      :lines: 1-9,11,14-
+      :linenos:
+      :emphasize-lines: 9, 14-15
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **9**
+        - Update the path if your requirements file is in a different location
+      * - **13-14**
+        - Modify the copy commands for the output formats you deploy
+
+.. dropdown:: CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
+
+   .. literalinclude:: ../../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 9, 15
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **9**
+        - Update the ``GIT_TAG`` to use a different version or commit hash
+      * - **15**
+        - Change if your docs subdirectory has a different location
+
+.. dropdown:: docs/CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
+
+   .. literalinclude:: ../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 5-7
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **5-7**
+        - Add or remove calls based on which output formats you need
+
+
+.. dropdown:: CMakePresets.json
+   :chevron: down-up
+
+   Defines presets for configuring and building documentation:
+
+   - **Configure Presets:** Sets up the build directory.
+   - **Build Presets:** Defines build formats individually and all in parallel.
+   - **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
+
+   .. literalinclude:: ../../CMakePresets.json
+      :language: json
+      :linenos:
+      :emphasize-lines: 18-23, 24-29, 34
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **18-23**
+        - Remove this preset to disable Markdown documentation builds
+      * - **24-29**
+        - Remove this preset to disable reStructuredText documentation builds
+      * - **34**
+        - Modify the targets list to build only the output formats you need in parallel
+
+Usage
+~~~~~
+
+To build documentation locally using CMake:
+
+.. code-block:: console
+
+   # Run the full workflow (configure + build all formats)
+   cmake --workflow --preset documentation-workflow
+
+   # Or configure and build separately
+   cmake --preset documentation
+   cmake --build --preset html        # Build HTML only
+   cmake --build --preset docs-parallel  # Build all formats
+
 .. _integration_examples:
 
 Integration Examples
@@ -285,6 +457,7 @@ Here's a complete example showing multiple :doc:`configuration-values`:
    This is a comprehensive documentation set for our project.
    It includes API references, usage examples, and tutorials.
    """
+   llms_txt_uri_template = "{base_url}{docname}.md"
 
    # Path handling
    html_baseurl = "https://docs.example.com/"

docs/source/conf.py

@@ -15,12 +15,17 @@ import subprocess
 project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
+
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
 A Sphinx extension that generates a summary llms.txt file,written in Markdown,
 and a single combined documentation llms-full.txt file, written in reStructuredText.
 """
 
+# This doesn't seem to be supported
+# rst_file_suffix = ".html.rst"
+markdown_file_suffix = ".html.md"
+
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
     GIT_TAG_OUTPUT = subprocess.check_output(["git", "tag", "--points-at", "HEAD"])
@@ -49,6 +54,10 @@ extensions = [
     "sphinx.ext.intersphinx",
     "sphinx_contributors",
     "sphinx_llms_txt",
+    "sphinxcontrib.restbuilder",
+    "sphinx_inline_tabs",
+    "sphinx.ext.extlinks",
+    "sphinx_design",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -88,7 +97,7 @@ html_theme_options = {
     "source_directory": "docs/source/",
 }
 
-html_baseurl = "https://sphinx-llms-txt.readthedocs.org/"
+html_baseurl = "https://sphinx-llms-txt.readthedocs.org/en/latest/"
 
 
 # -- Options for HTMLHelp output ---------------------------------------------

docs/source/configuration-values.rst

@@ -58,6 +58,15 @@ Project Configuration Values
 
    .. versionadded:: 0.2.0
 
+.. confval:: llms_txt_uri_template
+
+   - **Type**: string or ``None``
+   - **Default**: ``None``
+   - **Description**: Template string for generating URIs in ``llms.txt``.
+     See :ref:`customizing_uri_links`.
+
+   .. versionadded:: 0.7.0
+
 .. confval:: llms_txt_directives
 
    - **Type**: list of strings

docs/source/getting-started.rst

@@ -4,15 +4,17 @@ Getting Started
 Installation
 ------------
 
-Directly install via ``pip`` by using:
+Directly install by using:
 
-.. code-block:: bash
+.. tab:: via pip
+
+   .. code-block:: bash
 
       pip install sphinx-llms-txt
 
-Or with ``conda`` via ``conda-forge``:
+.. tab:: via conda:
 
-.. code::
+   .. code-block:: bash
 
       conda install -c conda-forge sphinx-llms-txt
 
@@ -32,7 +34,59 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
     sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
     sphinx-llms-txt: created /path/to/_build/html/llms.txt
 
+.. _choosing-output-format:
 
-.. tip:: Make sure to confirm the accuracy of the output files after installs and upgrades.
+Choosing an Output Format
+-------------------------
+
+By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
+For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
+
+.. list-table:: Output Format Comparison
+   :header-rows: 1
+   :widths: 18 27 27 27
+
+   * -
+     - Default
+     - Markdown
+     - reStructuredText
+   * - **Setup**
+     - No config
+     - CMake [#sphinxllm]_
+     - CMake
+   * - **Builder**
+     - Native [#native]_
+     - `sphinx-markdown-builder`_
+     - `sphinxcontrib-restbuilder`_
+   * - **Format**
+     - Raw reStructuredText source
+     - Rendered Markdown [#rendered]_
+     - Rendered reStructuredText [#rendered]_
+   * - **LLM Readability**
+     - Good - preserves structure for simple syntax
+     - Excellent - native LLM format
+     - Good - Can provide more structured content
+   * - **Key Advantage**
+     - Zero setup required
+     - More compact (less input tokens)
+     - Can preserve Sphinx semantics
+   * - **Key Disadvantage**
+     - Raw directives won't be parsed [#autodoc]_
+     - Loses structure from complex directives
+     - Can lose structure from complex directives
+   * - **llms-full.txt support**
+     - Supported with above caveats
+     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
+     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
+
+.. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
+.. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
+
+.. rubric:: Footnotes
+
+.. [#sphinxllm] See `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
+.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
+.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
+.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
+.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
 
-See :doc:`advanced-configuration` for more information about how to use **sphinx-llms-txt**.

docs/source/index.rst

@@ -8,21 +8,23 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
 Demo
 ----
 
-You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example.
+This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
+
+Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
 
 Highlights
 ----------
 
-1. **Content Collection**: Quickly gathers content from _sources, without needing a separate build
-2. **Directive Processing**: Resolves ``include`` directives by automatically incorporating their content
-3. **Path Resolution**: Transforms relative paths in directives to full paths
-4. **Output Generation**: Creates two optional files:
+**Zero Configuration**
+   Add the extension to your ``conf.py`` and you're done.
+   The extension automatically collects your documentation and generates both ``llms.txt`` and ``llms-full.txt`` during your normal Sphinx build.
 
-   - ``llms.txt``: A concise summary of your documentation, in Markdown
-   - ``llms-full.txt``: A comprehensive version with all documentation content, in reStructuredText
+**Intelligent Content Processing**
+   Automatically resolves ``include`` directives, transforms relative paths, and handles your documentation structure without manual intervention.
 
-5. **Content Filtering**: Allows you to exclude specific pages or sections
-6. **Source Code**: Allows you to include specific source code files
+**Customizable When Needed**
+   Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
+   See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
 
 .. toctree::
    :maxdepth: 2
@@ -36,6 +38,8 @@ Highlights
 
 .. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
 .. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
+.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
+.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
 .. _Sphinx: http://sphinx-doc.org/
 
 .. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg

pyproject.toml

@@ -48,6 +48,8 @@ test = [
     "pytest>=7.0.0",
 ]
 
+[tool.setuptools]
+  packages = ["sphinx_llms_txt"]
 
 [tool.setuptools.dynamic]
 version = {attr = "sphinx_llms_txt.__version__"}

sphinx_llms_txt/__init__.py

@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.6.0"
+__version__ = "0.7.2"
 
 # Export classes needed by tests
 __all__ = [
@@ -85,6 +85,7 @@ def build_finished(app: Sphinx, exception):
         config = {
             "llms_txt_file": app.config.llms_txt_file,
             "llms_txt_filename": app.config.llms_txt_filename,
+            "llms_txt_uri_template": app.config.llms_txt_uri_template,
             "llms_txt_title": app.config.llms_txt_title,
             "llms_txt_summary": summary,
             "llms_txt_full_file": app.config.llms_txt_full_file,
@@ -115,6 +116,7 @@ def setup(app: Sphinx) -> Dict[str, Any]:
 
     app.add_config_value("llms_txt_file", True, "env")
     app.add_config_value("llms_txt_filename", "llms.txt", "env")
+    app.add_config_value("llms_txt_uri_template", None, "env")
     app.add_config_value("llms_txt_full_file", True, "env")
     app.add_config_value("llms_txt_full_filename", "llms-full.txt", "env")
     app.add_config_value("llms_txt_full_max_size", None, "env")

sphinx_llms_txt/manager.py

@@ -223,6 +223,7 @@ class LLMSFullManager:
                     filtered_page_order,
                     self.collector.page_titles,
                     0,  # No line count since no llms-full.txt
+                    sources_dir,
                 )
 
             # Only warn if user explicitly wants llms-full.txt
@@ -523,6 +524,7 @@ class LLMSFullManager:
                         filtered_page_order,
                         self.collector.page_titles,
                         total_line_count,
+                        sources_dir,
                     )
                 return
             elif action == "note":
@@ -536,6 +538,7 @@ class LLMSFullManager:
                         filtered_page_order,
                         self.collector.page_titles,
                         total_line_count,
+                        sources_dir,
                     )
                 return
             elif action == "keep":
@@ -554,7 +557,10 @@ class LLMSFullManager:
         if success and self.config.get("llms_txt_file"):
             filtered_page_order = self._filter_ignored_pages(page_order)
             self.writer.write_verbose_info_to_file(
-                filtered_page_order, self.collector.page_titles, total_line_count
+                filtered_page_order,
+                self.collector.page_titles,
+                total_line_count,
+                sources_dir,
             )
 
     def _read_source_file(self, file_path: Path, docname: str) -> Tuple[str, int]:

sphinx_llms_txt/processor.py

@@ -128,8 +128,11 @@ class DocumentProcessor:
         Returns:
             Processed content with directive paths properly resolved
         """
+        # Get code block ranges to skip directives inside them
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Get the configured path directives to process
-        default_path_directives = ["image", "figure"]
+        default_path_directives = ["image", "figure", "literalinclude"]
         custom_path_directives = self.config.get("llms_txt_directives")
         path_directives = set(default_path_directives + custom_path_directives)
 
@@ -143,6 +146,11 @@ class DocumentProcessor:
         is_test = "pytest" in str(source_path) and "subdir" in str(source_path)
 
         def replace_directive_path(match, base_url=base_url, is_test=is_test):
+            # Check if this directive is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This directive is inside a code block, don't process it
+                return match.group(0)
+
             prefix = match.group(1)  # The entire directive prefix including whitespace
             path = match.group(3).strip()  # The path argument
 
@@ -276,6 +284,71 @@ class DocumentProcessor:
 
         return possible_paths
 
+    def _get_code_block_ranges(self, content: str) -> List[Tuple[int, int]]:
+        """Find all code block ranges in the content.
+
+        Args:
+            content: The source content to analyze
+
+        Returns:
+            List of (start, end) tuples representing code block character
+            ranges
+        """
+        code_block_ranges = []
+
+        # Match code block as well as `code` and `sourcecode` aliases
+        code_block_pattern = re.compile(
+            r"^(\s*)\.\.\s+(code-block|code|sourcecode)::\s*\S*\s*$", re.MULTILINE
+        )
+
+        for match in code_block_pattern.finditer(content):
+            start_pos = match.start()
+            indent = match.group(1)
+            indent_len = len(indent)
+
+            # Find the end of the code block by looking for the next line
+            # that is not indented more than the directive
+            block_start = match.end()
+            pos = block_start
+
+            # Skip any blank lines immediately after the directive
+            while pos < len(content) and content[pos] in "\n":
+                pos += 1
+
+            # Find where the code block ends
+            lines = content[pos:].split("\n")
+            block_end = pos
+            for line in lines:
+                if line.strip():  # Non-empty line
+                    # Check indentation level
+                    line_indent = len(line) - len(line.lstrip())
+                    if line_indent <= indent_len:
+                        # The block ends when we find a line that is indented
+                        # less than the directive itself
+                        break
+                block_end += len(line) + 1  # +1 for the newline
+
+            code_block_ranges.append((start_pos, block_end))
+
+        return code_block_ranges
+
+    def _is_in_code_block(
+        self, match_start: int, code_block_ranges: List[Tuple[int, int]]
+    ) -> bool:
+        """Check if a match position is within a code block.
+
+        Args:
+            match_start: The starting position of the match
+            code_block_ranges: List of (start, end) tuples for code blocks
+
+        Returns:
+            True if the match is within a code block, False otherwise
+        """
+        for block_start, block_end in code_block_ranges:
+            if block_start <= match_start < block_end:
+                return True
+        return False
+
     def _process_includes(self, content: str, source_path: Path) -> str:
         """Process include directives in content.
 
@@ -286,11 +359,18 @@ class DocumentProcessor:
         Returns:
             Processed content with include directives replaced with included content
         """
+        code_block_ranges = self._get_code_block_ranges(content)
+
         # Find all include directives using regex
         include_pattern = build_directive_pattern(["include"])
 
         # Function to replace each include with content
         def replace_include(match):
+            # Check if this include is within a code block
+            if self._is_in_code_block(match.start(), code_block_ranges):
+                # This include is inside a code block, don't process it
+                return match.group(0)
+
             include_path = match.group(3)
             directive_part = match.group(
                 1

sphinx_llms_txt/writer.py

@@ -11,6 +11,41 @@ from sphinx.util import logging
 logger = logging.getLogger(__name__)
 
 
+def _fix_mojibake(text: str) -> str:
+    """Fix common UTF-8/Windows-1252 mojibake in text.
+
+    Mojibake (文字化け, "character transformation") is a Japanese term for garbled text
+    caused by decoding bytes with the wrong character encoding.
+
+    This handles the case where UTF-8 bytes were incorrectly decoded as Windows-1252
+    (or Latin-1), resulting in corrupted characters like:
+    - ' (U+2019) becoming ’
+    - " (U+201C) becoming “
+    - " (U+201D) becoming â€
+    - — (U+2014) becoming â€"
+    - – (U+2013) becoming â€"
+    - … (U+2026) becoming …
+
+    Args:
+        text: The potentially corrupted text string
+
+    Returns:
+        The repaired text string, or the original if no repair was needed/possible
+    """
+    if not text:
+        return text
+
+    try:
+        # Try to encode the text as Windows-1252 (which would succeed if it contains
+        # the mojibake characters) and then decode as UTF-8 (to get the original)
+        return text.encode("windows-1252").decode("utf-8")
+    except (UnicodeDecodeError, UnicodeEncodeError):
+        # If encoding/decoding fails, the text is either:
+        # - Already correct UTF-8
+        # - Corrupted in a different way we can't fix
+        return text
+
+
 class FileWriter:
     """Handles writing processed content to output files."""
 
@@ -19,6 +54,42 @@ class FileWriter:
         self.outdir = outdir
         self.app = app
 
+    def _resolve_uri_template(self, sources_dir: Path = None) -> str:
+        """Resolve which URI template to use based on configuration and sources_dir.
+
+        Args:
+            sources_dir: Path to _sources directory (None if not found)
+
+        Returns:
+            The template string to use for generating URIs
+        """
+        # If custom template exists
+        custom_template = self.config.get("llms_txt_uri_template")
+
+        if custom_template:
+            # Validate user's template by checking for valid variable names
+            try:
+                # Try formatting with test valid values to validate syntax
+                test_values = {
+                    "base_url": "http://example.com/",
+                    "docname": "test",
+                    "suffix": ".rst",
+                    "sourcelink_suffix": ".txt",
+                }
+                custom_template.format(**test_values)
+                return custom_template
+            except (KeyError, ValueError) as e:
+                logger.warning(
+                    f"sphinx-llms-txt: Invalid llms_txt_uri_template: {e}. "
+                    f"Falling back to default."
+                )
+
+        # Else, use one of the default templates
+        if sources_dir:
+            return "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+        else:
+            return "{base_url}{docname}.html"
+
     def write_combined_file(
         self, content_parts: List[str], output_path: Path, total_line_count: int
     ) -> bool:
@@ -50,6 +121,7 @@ class FileWriter:
         page_order: Union[List[str], List[Tuple[str, str]]],
         page_titles: Dict[str, str],
         total_line_count: int = 0,
+        sources_dir: Path = None,
     ) -> bool:
         """Write summary information to the llms.txt file.
 
@@ -57,6 +129,7 @@ class FileWriter:
             page_order: Ordered list of document names or (docname, suffix) tuples
             page_titles: Dictionary mapping docnames to titles
             total_line_count: Total number of lines in the combined content
+            sources_dir: Path to _sources directory (None if not found)
 
         Returns:
             True if successful, False otherwise
@@ -81,6 +154,8 @@ class FileWriter:
                     and hasattr(self.app.config, "project")
                 ):
                     project_name = self.app.config.project
+                # Fix any UTF-8/Windows-1252 mojibake in the project name
+                project_name = _fix_mojibake(project_name)
                 f.write(f"# {project_name}\n\n")
 
                 # Add description if available
@@ -89,6 +164,8 @@ class FileWriter:
                     # Trim leading and trailing whitespace
                     description = description.strip()
                     if description:
+                        # Fix any UTF-8/Windows-1252 mojibake in the description
+                        description = _fix_mojibake(description)
                         # Only add blockquote if description is not empty
                         # Replace newlines with newline + blockquote marker to maintain
                         # blockquote formatting
@@ -102,14 +179,39 @@ class FileWriter:
                 if not base_url.endswith("/"):
                     base_url += "/"
 
+                # Get sourcelink suffix from Sphinx config
+                sourcelink_suffix = ""
+                if self.app and hasattr(self.app.config, "html_sourcelink_suffix"):
+                    sourcelink_suffix = self.app.config.html_sourcelink_suffix
+                    # Handle empty string case specially
+                    if sourcelink_suffix == "":
+                        sourcelink_suffix = ""  # Keep it empty
+                    elif not sourcelink_suffix.startswith("."):
+                        sourcelink_suffix = "." + sourcelink_suffix
+
+                # Resolve which template to use
+                uri_template = self._resolve_uri_template(sources_dir)
+
                 for item in page_order:
                     # Handle both old format (str) and new format (tuple)
                     if isinstance(item, tuple):
-                        docname, _ = item
+                        docname, suffix = item
                     else:
                         docname = item
+                        suffix = None
+
                     title = page_titles.get(docname, docname)
-                    f.write(f"- [{title}]({base_url}{docname}.html)\n")
+                    # Fix any UTF-8/Windows-1252 mojibake in the title
+                    title = _fix_mojibake(title)
+
+                    uri = uri_template.format(
+                        base_url=base_url,
+                        docname=docname,
+                        suffix=suffix or "",
+                        sourcelink_suffix=sourcelink_suffix,
+                    )
+
+                    f.write(f"- [{title}]({uri})\n")
 
             logger.info(f"sphinx-llms-txt: created {output_path}")
             return True

tests/test_llms_txt.py

@@ -198,6 +198,31 @@ def test_process_includes(tmp_path):
     assert processed_content == expected_content
 
 
+def test_process_includes_in_code_block(tmp_path):
+    """Test that an `include` within a `code-block` is not processed."""
+    # Create a processor
+    config = {"llms_txt_directives": []}
+    processor = DocumentProcessor(config)
+
+    # Create a source file that uses include syntax within a `code-block`
+    source_content = (
+        "Normal paragraph.\n\n"
+        ".. code-block:: rst\n\n"
+        "   .. include:: foo.txt\n\n"
+        "Another normal paragraph."
+    )
+    source_file = tmp_path / "source.txt"
+    with open(source_file, "w", encoding="utf-8") as f:
+        f.write(source_content)
+
+    # Run the include directive processor
+    processed_content = processor._process_includes(source_content, source_file)
+
+    # Check that the include directive was not processed
+    expected_content = source_content
+    assert processed_content == expected_content
+
+
 def test_process_includes_with_relative_paths(tmp_path):
     """Test that include directives with relative paths are processed correctly."""
     # Create a processor
@@ -794,6 +819,7 @@ def test_summary_default_uses_first_paragraph():
             llms_txt_summary = None  # Not configured
             llms_txt_file = True
             llms_txt_filename = "llms.txt"
+            llms_txt_uri_template = None
             llms_txt_title = None
             llms_txt_full_file = True
             llms_txt_full_filename = "llms-full.txt"
@@ -1156,3 +1182,48 @@ def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
     # Verify llms.txt was still created
     llms_txt = outdir / "llms.txt"
     assert llms_txt.exists()
+
+
+def test_fix_mojibake():
+    """
+    Test that the _fix_mojibake function correctly repairs UTF-8/Windows-1252 mojibake.
+    """
+    from sphinx_llms_txt.writer import _fix_mojibake
+
+    # Test case from issue #65: smart apostrophe corrupted
+    # U+2019 (') encoded as UTF-8 (E2 80 99) then decoded as Windows-1252 gives ’
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 99->™(U+2122)
+    corrupted = "What\u00e2\u20ac\u2122s New"  # ’
+    expected = "What\u2019s New"  # ' = U+2019
+    assert _fix_mojibake(corrupted) == expected
+
+    # Test left double quote: U+201C (") -> “
+    # U+201C encoded as UTF-8: E2 80 9C
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 9C->œ(U+0153)
+    corrupted_ldq = "He said \u00e2\u20ac\u0153Hello"
+    expected_ldq = "He said \u201cHello"
+    assert _fix_mojibake(corrupted_ldq) == expected_ldq
+
+    # Test em dash: U+2014 (—) -> â€"
+    # U+2014 encoded as UTF-8: E2 80 94
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 94->"(U+201D)
+    corrupted_emdash = "one\u00e2\u20ac\u201dtwo"
+    expected_emdash = "one\u2014two"
+    assert _fix_mojibake(corrupted_emdash) == expected_emdash
+
+    # Test that already correct text is not modified
+    # Using Unicode escape for smart apostrophe
+    correct = "What\u2019s New In Our Latest Release!"
+    assert _fix_mojibake(correct) == correct
+
+    # Test empty string
+    assert _fix_mojibake("") == ""
+
+    # Test plain ASCII text passes through unchanged
+    plain = "Hello World"
+    assert _fix_mojibake(plain) == plain
+
+    # Test mixed mojibake and normal text
+    mixed = "Here\u00e2\u20ac\u2122s a test"
+    expected_mixed = "Here\u2019s a test"
+    assert _fix_mojibake(mixed) == expected_mixed

tests/test_uri_template.py

@@ -0,0 +1,175 @@
+"""Test URI template functionality for llms.txt links."""
+
+from sphinx_llms_txt import FileWriter
+
+
+def test_uri_template_with_sources_dir(tmp_path):
+    """Test that default template uses _sources links when sources_dir exists."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    # Create _sources directory to simulate its existence
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": (
+            "{base_url}_sources/{docname}{suffix}{sourcelink_suffix}"
+        ),
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+        "about": "About Us",
+    }
+
+    # Page order with suffixes (simulating _sources files exist)
+    page_order = [("index", ".rst"), ("about", ".md")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    # Check that the file was created
+    verbose_file = build_dir / "llms.txt"
+    assert verbose_file.exists()
+
+    # Read the file content
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should link to _sources files
+    assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content
+    assert "- [About Us](https://example.com/_sources/about.md.txt)" in content
+
+
+def test_uri_template_without_sources_dir(tmp_path):
+    """
+    Test that HTML template is used when sources_dir doesn't exist and no custom
+    template.
+    """
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        # No custom template set
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir))
+
+    page_titles = {
+        "index": "Home Page",
+        "about": "About Us",
+    }
+
+    # Page order without suffixes (simulating no _sources)
+    page_order = [("index", None), ("about", None)]
+
+    # Pass None for sources_dir to simulate it doesn't exist
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, None)
+
+    # Check that the file was created
+    verbose_file = build_dir / "llms.txt"
+    assert verbose_file.exists()
+
+    # Read the file content
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should fallback to HTML links
+    assert "- [Home Page](https://example.com/index.html)" in content
+    assert "- [About Us](https://example.com/about.html)" in content
+
+
+def test_uri_template_custom(tmp_path):
+    """Test that custom URI template works correctly."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    # Custom template that uses different path
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": "{base_url}raw/{docname}{suffix}",
+        "html_baseurl": "https://example.com/",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+    }
+
+    page_order = [("index", ".rst")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    verbose_file = build_dir / "llms.txt"
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should use custom template
+    assert "- [Home Page](https://example.com/raw/index.rst)" in content
+
+
+def test_uri_template_invalid_fallback(tmp_path):
+    """
+    Test that invalid template falls back to default sources template when
+    sources_dir exists.
+    """
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    # Mock app with html_sourcelink_suffix
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    # Invalid template with typo in variable name
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "llms_txt_uri_template": "{base_urll}/{docname}",
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+    }
+
+    page_order = [("index", ".rst")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    verbose_file = build_dir / "llms.txt"
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should fallback to default sources template
+    assert "- [Home Page](https://example.com/_sources/index.rst.txt)" in content