Fix encoding of Unicode characters

Bumps the all-github-actions group with 1 update: [actions/checkout](https://github.com/actions/checkout).


Updates `actions/checkout` from 5 to 6
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v5...v6)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
  dependency-group: all-github-actions
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/test.yml

@@ -10,7 +10,7 @@ jobs:
   pre-commit:
       runs-on: ubuntu-latest
       steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - name: Set up Python 3.10
         uses: actions/setup-python@v6
         with:
@@ -23,7 +23,7 @@ jobs:
         python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6

.readthedocs.yml

@@ -9,6 +9,8 @@ build:
     - pip install -r docs/requirements.txt
     - pip install -e .
     - cmake --workflow --preset documentation-workflow
+    # Generate llms.txt variants for demo purposes
+    - python docs/generate_llms_variants.py build/html
     # Copy built documentation to Read the Docs output directory
     - mkdir -p $READTHEDOCS_OUTPUT/html
     - cp -r build/html/* $READTHEDOCS_OUTPUT/html/

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Changelog
 =========
 
+0.7.2
+-----
+
+- Fix encoding of Unicode characters (smart quotes, em dashes, etc.) in llms.txt output
+  `#65 <https://github.com/jdillard/sphinx-llms-txt/issues/65>`_
+
+0.7.1
+-----
+
+- Don't process includes within code blocks
+
 0.7.0
 -----
 

CMakeLists.txt

@@ -1,8 +1,15 @@
 cmake_minimum_required(VERSION 3.15)
-project(SphinxLLMsTxt VERSION 1.0.0 LANGUAGES NONE)
+project(SphinxDocs VERSION 1.0.0 LANGUAGES NONE)
 
-# Add CMake module path
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+# Fetch Sphinx CMake modules
+include(FetchContent)
+FetchContent_Declare(
+  sphinx_cmake_modules
+  GIT_REPOSITORY https://github.com/jdillard/sphinx-cmake-modules.git
+  GIT_TAG        main
+)
+FetchContent_MakeAvailable(sphinx_cmake_modules)
+list(APPEND CMAKE_MODULE_PATH "${sphinx_cmake_modules_SOURCE_DIR}/cmake/modules")
 
 # Add documentation
 add_subdirectory(docs)

CMakePresets.json

@@ -29,7 +29,7 @@
     },
     {
       "name": "docs-parallel",
-      "displayName": "Build HTML and Markdown in parallel",
+      "displayName": "Build all output formats in parallel",
       "configurePreset": "documentation",
       "targets": ["html", "markdown", "rst"]
     }

cmake/SphinxUtils.cmake

@@ -1,28 +0,0 @@
-# Sphinx related utilities
-
-set(SPHINX_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/source)
-set(SPHINX_BUILD ${CMAKE_BINARY_DIR})
-
-# Function to find Sphinx in the system
-function(setup_sphinx_environment)
-    # Find sphinx-build executable in system
-    find_program(SPHINX_EXECUTABLE
-        NAMES sphinx-build
-        DOC "Sphinx documentation generator"
-    )
-
-    if(NOT SPHINX_EXECUTABLE)
-        message(FATAL_ERROR "sphinx-build not found. Please install Sphinx.")
-    endif()
-
-    # Export to parent scope
-    set(SPHINX_EXECUTABLE "${SPHINX_EXECUTABLE}" PARENT_SCOPE)
-endfunction()
-
-# Function to add a Sphinx builder target
-function(add_sphinx_builder builder_name)
-    add_custom_target(${builder_name}
-        COMMAND ${SPHINX_EXECUTABLE} -b ${builder_name} ${SPHINX_SOURCE} ${SPHINX_BUILD}/${builder_name}
-        VERBATIM
-    )
-endfunction()

docs/generate_llms_variants.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Generate variant llms.txt files for demo purposes.
+
+Takes the generated llms.txt (with _sources links) and creates:
+- llms.txt - default with _sources links (unchanged)
+- llms.md.txt - .html.md links
+- llms.rst.txt - .rst links
+"""
+
+import re
+import sys
+from pathlib import Path
+
+
+def get_base_url() -> str:
+    """Import base_url from conf.py."""
+    sys.path.insert(0, str(Path(__file__).parent / "source"))
+    from conf import html_baseurl  # noqa: E402
+
+    return html_baseurl
+
+
+def generate_variants(build_dir: Path) -> None:
+    """Generate llms.txt variants from the original file."""
+    original = build_dir / "llms.txt"
+
+    if not original.exists():
+        print(f"Error: {original} not found")
+        sys.exit(1)
+
+    content = original.read_text()
+    base_url = get_base_url()
+
+    # Pattern to match links like: https://.../_sources/{docname}.rst.txt
+    link_pattern = re.compile(
+        rf"({re.escape(base_url)})_sources/([a-zA-Z0-9_/\-]+)\.rst\.txt"
+    )
+
+    # Generate .html.md variant
+    md_content = link_pattern.sub(r"\1\2.html.md", content)
+    (build_dir / "llms.md.txt").write_text(md_content)
+    print(f"Generated: {build_dir / 'llms.md.txt'} (.html.md links)")
+
+    # Generate .rst variant
+    rst_content = link_pattern.sub(r"\1\2.rst", content)
+    (build_dir / "llms.rst.txt").write_text(rst_content)
+    print(f"Generated: {build_dir / 'llms.rst.txt'} (.rst links)")
+
+    print(f"Kept: {build_dir / 'llms.txt'} (_sources links)")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        build_dir = Path(sys.argv[1])
+    else:
+        build_dir = Path("build/html")
+
+    generate_variants(build_dir)

docs/requirements.txt

@@ -2,6 +2,7 @@ furo
 esbonio
 sphinx-contributors
 sphinx
+sphinx-design
 sphinx-llms-txt
 sphinx-inline-tabs
 sphinxext-opengraph

docs/source/advanced-configuration.rst

@@ -303,42 +303,121 @@ CMake Workflow
 ^^^^^^^^^^^^^^
 
 This project uses CMake to orchestrate documentation builds across multiple output formats, serving as a simple demo of the functionality.
-Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
 This approach enables parallel builds and integrates well with CI/CD platforms like Read the Docs.
 
+Building multiple formats allows you to compare what works best for your docs, as well as allows users to choose which format to feed to their LLM.
+Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
+
 Key Files
 ~~~~~~~~~
 
+These configuration files serve as a simple example of a Sphinx site hosted on Read The Docs, some modification may be needed.
+
 .. code-block:: text
 
    .
    ├── .readthedocs.yml
    ├── CMakeLists.txt
    ├── CMakePresets.json
-   ├── cmake/
-   │   └── SphinxUtils.cmake
    └── docs/
        └── CMakeLists.txt
 
-:ghfile:`.readthedocs.yml`
+Each section below contains a summary of the file's purpose, the full contents of the file, and a table describing key lines that may need modification.
+
+.. dropdown:: .readthedocs.yml
+   :chevron: down-up
+
    A Read The Docs config file that installs dependencies, then runs the full documentation workflow which builds all output formats in parallel, and copies them into a single deploy location.
 
-:ghfile:`CMakeLists.txt`
-   A CMake config file that sets up the project and includes the ``cmake/`` module path.
+   .. literalinclude:: ../../.readthedocs.yml
+      :language: yaml
+      :lines: 1-9,11,14-
+      :linenos:
+      :emphasize-lines: 9, 14-15
 
-:ghfile:`docs/CMakeLists.txt`
-   A CMake config file that includes the Sphinx utilities and defines the documentation-specific build targets.
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
 
-:ghfile:`cmake/SphinxUtils.cmake`
-   A CMake module that provides Sphinx related utilities.
+      * - Line
+        - Description
+      * - **9**
+        - Update the path if your requirements file is in a different location
+      * - **13-14**
+        - Modify the copy commands for the output formats you deploy
+
+.. dropdown:: CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that sets up the project, fetches the shared `sphinx-cmake-modules <https://github.com/jdillard/sphinx-cmake-modules>`_, and includes the docs subdirectory.
+
+   .. literalinclude:: ../../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 9, 15
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **9**
+        - Update the ``GIT_TAG`` to use a different version or commit hash
+      * - **15**
+        - Change if your docs subdirectory has a different location
+
+.. dropdown:: docs/CMakeLists.txt
+   :chevron: down-up
+
+   A CMake config file that includes the `SphinxUtils <https://github.com/jdillard/sphinx-cmake-modules/blob/v0.1.0/SphinxUtils.cmake>`_ module from FetchContent and defines the documentation-specific build targets.
+
+   .. literalinclude:: ../CMakeLists.txt
+      :language: cmake
+      :linenos:
+      :emphasize-lines: 5-7
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **5-7**
+        - Add or remove calls based on which output formats you need
+
+
+.. dropdown:: CMakePresets.json
+   :chevron: down-up
 
-:ghfile:`CMakePresets.json`
    Defines presets for configuring and building documentation:
 
    - **Configure Presets:** Sets up the build directory.
-   - **Build Presets:** Defines Build formats individually and all in parallel.
+   - **Build Presets:** Defines build formats individually and all in parallel.
    - **Workflow Presets:** Runs the configure preset followed by the parallel build preset.
 
+   .. literalinclude:: ../../CMakePresets.json
+      :language: json
+      :linenos:
+      :emphasize-lines: 18-23, 24-29, 34
+
+   .. list-table::
+      :header-rows: 1
+      :width: 100%
+      :widths: 15 85
+
+      * - Line
+        - Description
+      * - **18-23**
+        - Remove this preset to disable Markdown documentation builds
+      * - **24-29**
+        - Remove this preset to disable reStructuredText documentation builds
+      * - **34**
+        - Modify the targets list to build only the output formats you need in parallel
+
 Usage
 ~~~~~
 

docs/source/conf.py

@@ -16,7 +16,6 @@ project = "sphinx-llms-txt"
 copyright = "Jared Dillard"
 author = "Jared Dillard"
 
-llms_txt_uri_template = "{base_url}{docname}.md"
 llms_txt_code_files = ["+:../../sphinx_llms_txt/*.py"]
 llms_txt_summary = """
 A Sphinx extension that generates a summary llms.txt file,written in Markdown,
@@ -25,10 +24,7 @@ and a single combined documentation llms-full.txt file, written in reStructuredT
 
 # This doesn't seem to be supported
 # rst_file_suffix = ".html.rst"
-
-extlinks = {
-    "ghfile": ("https://github.com/jdillard/sphinx-llms-txt/blob/main/%s", "%s")
-}
+markdown_file_suffix = ".html.md"
 
 # check if the current commit is tagged as a release (vX.Y.Z)
 try:
@@ -61,6 +57,7 @@ extensions = [
     "sphinxcontrib.restbuilder",
     "sphinx_inline_tabs",
     "sphinx.ext.extlinks",
+    "sphinx_design",
 ]
 
 # The language for content autogenerated by Sphinx. Refer to documentation

docs/source/getting-started.rst

@@ -34,24 +34,34 @@ After the HTML finishes building, **sphinx-llms-txt** will output the location o
     sphinx-llms-txt: Created /path/to/_build/html/llms-full.txt with 45 sources and 6879 lines
     sphinx-llms-txt: created /path/to/_build/html/llms.txt
 
+.. _choosing-output-format:
+
 Choosing an Output Format
 -------------------------
 
-By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files in :confval:`_sources/ <sphinx:html_copy_source>`.
-For optimal LLM support, you can use `sphinx-markdown-builder`_ and/or `sphinxcontrib-restbuilder`_, set up in parallel builds using :ref:`CMake <cmake_workflow>`.
+By default, **sphinx-llms-txt** requires no additional configuration and links to raw reStructuredText source files created by the HTML builder.
+For optimal LLM support, see the alternative builders below and the :ref:`CMake workflow <cmake_workflow>` for setup.
 
 .. list-table:: Output Format Comparison
    :header-rows: 1
    :widths: 18 27 27 27
 
    * -
-     - Default (no config)
-     - Markdown (CMake)
-     - RST (CMake)
+     - Default
+     - Markdown
+     - reStructuredText
+   * - **Setup**
+     - No config
+     - CMake [#sphinxllm]_
+     - CMake
+   * - **Builder**
+     - Native [#native]_
+     - `sphinx-markdown-builder`_
+     - `sphinxcontrib-restbuilder`_
    * - **Format**
-     - Raw RST source
-     - Rendered Markdown
-     - Rendered RST
+     - Raw reStructuredText source
+     - Rendered Markdown [#rendered]_
+     - Rendered reStructuredText [#rendered]_
    * - **LLM Readability**
      - Good - preserves structure for simple syntax
      - Excellent - native LLM format
@@ -61,18 +71,22 @@ For optimal LLM support, you can use `sphinx-markdown-builder`_ and/or `sphinxco
      - More compact (less input tokens)
      - Can preserve Sphinx semantics
    * - **Key Disadvantage**
-     - Raw directives (e.g., autodoc) won't be parsed
+     - Raw directives won't be parsed [#autodoc]_
      - Loses structure from complex directives
      - Can lose structure from complex directives
    * - **llms-full.txt support**
-     - Suported with above caveats
-     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__
-     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__
-
-
-See :ref:`cmake_workflow` for an example of building HTML, Markdown, and RST in parallel.
-Use :confval:`llms_txt_uri_template` to configure links to point to your preferred format.
+     - Supported with above caveats
+     - Pending `support <https://github.com/liran-funaro/sphinx-markdown-builder/pull/37>`__ [#pending]_
+     - Pending `support <https://github.com/sphinx-contrib/restbuilder/pull/35>`__ [#pending]_
 
 .. _sphinx-markdown-builder: https://pypi.org/project/sphinx-markdown-builder/
 .. _sphinxcontrib-restbuilder: https://pypi.org/project/sphinxcontrib-restbuilder/
 
+.. rubric:: Footnotes
+
+.. [#sphinxllm] See `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ as an alternative for CMake-free Markdown builds.
+.. [#native] Uses raw :confval:`_sources/ <sphinx:html_copy_source>` files created by Sphinx's HTML builder with some minor enhancements.
+.. [#autodoc] Directives like ``autodoc`` will appear as raw directive syntax rather than the extracted docstrings.
+.. [#pending] PRs that add ``llms-full.txt`` concatenation support have yet to be released.
+.. [#rendered] Directives are expanded and processed before output, so content like autodoc docstrings will be included.
+

docs/source/index.rst

@@ -8,7 +8,9 @@ A `Sphinx`_ extension that generates a summary ``llms.txt`` file, written in Mar
 Demo
 ----
 
-You can see this Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as a simple example.
+This Sphinx project's `llms.txt`_ and `llms-full.txt`_ files as an example of the default output format.
+
+Alternative :ref:`output formats <choosing-output-format>` are also available. For example: `Markdown`_ and `reStructuredText`_.
 
 Highlights
 ----------
@@ -36,6 +38,8 @@ Highlights
 
 .. _llms.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.txt
 .. _llms-full.txt: https://sphinx-llms-txt.readthedocs.io/en/latest/llms-full.txt
+.. _Markdown: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.md.txt
+.. _reStructuredText: https://sphinx-llms-txt.readthedocs.io/en/latest/llms.rst.txt
 .. _Sphinx: http://sphinx-doc.org/
 
 .. |PyPI version| image:: https://img.shields.io/pypi/v/sphinx-llms-txt.svg

sphinx_llms_txt/__init__.py

@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.7.0"
+__version__ = "0.7.2"
 
 # Export classes needed by tests
 __all__ = [

sphinx_llms_txt/writer.py

@@ -11,6 +11,41 @@ from sphinx.util import logging
 logger = logging.getLogger(__name__)
 
 
+def _fix_mojibake(text: str) -> str:
+    """Fix common UTF-8/Windows-1252 mojibake in text.
+
+    Mojibake (文字化け, "character transformation") is a Japanese term for garbled text
+    caused by decoding bytes with the wrong character encoding.
+
+    This handles the case where UTF-8 bytes were incorrectly decoded as Windows-1252
+    (or Latin-1), resulting in corrupted characters like:
+    - ' (U+2019) becoming ’
+    - " (U+201C) becoming “
+    - " (U+201D) becoming â€
+    - — (U+2014) becoming â€"
+    - – (U+2013) becoming â€"
+    - … (U+2026) becoming …
+
+    Args:
+        text: The potentially corrupted text string
+
+    Returns:
+        The repaired text string, or the original if no repair was needed/possible
+    """
+    if not text:
+        return text
+
+    try:
+        # Try to encode the text as Windows-1252 (which would succeed if it contains
+        # the mojibake characters) and then decode as UTF-8 (to get the original)
+        return text.encode("windows-1252").decode("utf-8")
+    except (UnicodeDecodeError, UnicodeEncodeError):
+        # If encoding/decoding fails, the text is either:
+        # - Already correct UTF-8
+        # - Corrupted in a different way we can't fix
+        return text
+
+
 class FileWriter:
     """Handles writing processed content to output files."""
 
@@ -119,6 +154,8 @@ class FileWriter:
                     and hasattr(self.app.config, "project")
                 ):
                     project_name = self.app.config.project
+                # Fix any UTF-8/Windows-1252 mojibake in the project name
+                project_name = _fix_mojibake(project_name)
                 f.write(f"# {project_name}\n\n")
 
                 # Add description if available
@@ -127,6 +164,8 @@ class FileWriter:
                     # Trim leading and trailing whitespace
                     description = description.strip()
                     if description:
+                        # Fix any UTF-8/Windows-1252 mojibake in the description
+                        description = _fix_mojibake(description)
                         # Only add blockquote if description is not empty
                         # Replace newlines with newline + blockquote marker to maintain
                         # blockquote formatting
@@ -162,6 +201,8 @@ class FileWriter:
                         suffix = None
 
                     title = page_titles.get(docname, docname)
+                    # Fix any UTF-8/Windows-1252 mojibake in the title
+                    title = _fix_mojibake(title)
 
                     uri = uri_template.format(
                         base_url=base_url,

tests/test_llms_txt.py

@@ -1182,3 +1182,48 @@ def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
     # Verify llms.txt was still created
     llms_txt = outdir / "llms.txt"
     assert llms_txt.exists()
+
+
+def test_fix_mojibake():
+    """
+    Test that the _fix_mojibake function correctly repairs UTF-8/Windows-1252 mojibake.
+    """
+    from sphinx_llms_txt.writer import _fix_mojibake
+
+    # Test case from issue #65: smart apostrophe corrupted
+    # U+2019 (') encoded as UTF-8 (E2 80 99) then decoded as Windows-1252 gives ’
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 99->™(U+2122)
+    corrupted = "What\u00e2\u20ac\u2122s New"  # ’
+    expected = "What\u2019s New"  # ' = U+2019
+    assert _fix_mojibake(corrupted) == expected
+
+    # Test left double quote: U+201C (") -> “
+    # U+201C encoded as UTF-8: E2 80 9C
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 9C->œ(U+0153)
+    corrupted_ldq = "He said \u00e2\u20ac\u0153Hello"
+    expected_ldq = "He said \u201cHello"
+    assert _fix_mojibake(corrupted_ldq) == expected_ldq
+
+    # Test em dash: U+2014 (—) -> â€"
+    # U+2014 encoded as UTF-8: E2 80 94
+    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 94->"(U+201D)
+    corrupted_emdash = "one\u00e2\u20ac\u201dtwo"
+    expected_emdash = "one\u2014two"
+    assert _fix_mojibake(corrupted_emdash) == expected_emdash
+
+    # Test that already correct text is not modified
+    # Using Unicode escape for smart apostrophe
+    correct = "What\u2019s New In Our Latest Release!"
+    assert _fix_mojibake(correct) == correct
+
+    # Test empty string
+    assert _fix_mojibake("") == ""
+
+    # Test plain ASCII text passes through unchanged
+    plain = "Hello World"
+    assert _fix_mojibake(plain) == plain
+
+    # Test mixed mojibake and normal text
+    mixed = "Here\u00e2\u20ac\u2122s a test"
+    expected_mixed = "Here\u2019s a test"
+    assert _fix_mojibake(mixed) == expected_mixed