Fix double extension in llms.txt URLs when source suffix matches sourcelink suffix

CHANGELOG.rst

@@ -4,8 +4,8 @@ Changelog
 0.7.2
 -----
 
-- Fix encoding of Unicode characters (smart quotes, em dashes, etc.) in llms.txt output
+- Fix double extension in llms.txt URLs when source suffix matches sourcelink suffix
-  `#65 <https://github.com/jdillard/sphinx-llms-txt/issues/65>`_
+  `#63 <https://github.com/jdillard/sphinx-llms-txt/issues/63>`_
 
 0.7.1
 -----

docs/source/index.rst

@@ -26,6 +26,11 @@ Highlights
    Filter content, include source code files, or integrate with alternative output formats like Markdown for even better LLM compatibility.
    See :doc:`getting-started` for output format options and :doc:`configuration-values` for all settings.
 
+.. seealso::
+
+   For better default output without configuration, see `sphinx-llm <https://github.com/NVIDIA/sphinx-llm>`_ from NVIDIA.
+   sphinx-llms-txt is best when customized with alternative output formats, content filtering, or source code inclusion.
+
 .. toctree::
    :maxdepth: 2
 

sphinx_llms_txt/writer.py

@@ -11,41 +11,6 @@ from sphinx.util import logging
 logger = logging.getLogger(__name__)
 
 
-def _fix_mojibake(text: str) -> str:
-    """Fix common UTF-8/Windows-1252 mojibake in text.
-
-    Mojibake (文字化け, "character transformation") is a Japanese term for garbled text
-    caused by decoding bytes with the wrong character encoding.
-
-    This handles the case where UTF-8 bytes were incorrectly decoded as Windows-1252
-    (or Latin-1), resulting in corrupted characters like:
-    - ' (U+2019) becoming ’
-    - " (U+201C) becoming “
-    - " (U+201D) becoming â€
-    - — (U+2014) becoming â€"
-    - – (U+2013) becoming â€"
-    - … (U+2026) becoming …
-
-    Args:
-        text: The potentially corrupted text string
-
-    Returns:
-        The repaired text string, or the original if no repair was needed/possible
-    """
-    if not text:
-        return text
-
-    try:
-        # Try to encode the text as Windows-1252 (which would succeed if it contains
-        # the mojibake characters) and then decode as UTF-8 (to get the original)
-        return text.encode("windows-1252").decode("utf-8")
-    except (UnicodeDecodeError, UnicodeEncodeError):
-        # If encoding/decoding fails, the text is either:
-        # - Already correct UTF-8
-        # - Corrupted in a different way we can't fix
-        return text
-
-
 class FileWriter:
     """Handles writing processed content to output files."""
 
@@ -154,8 +119,6 @@ class FileWriter:
                     and hasattr(self.app.config, "project")
                 ):
                     project_name = self.app.config.project
-                # Fix any UTF-8/Windows-1252 mojibake in the project name
-                project_name = _fix_mojibake(project_name)
                 f.write(f"# {project_name}\n\n")
 
                 # Add description if available
@@ -164,8 +127,6 @@ class FileWriter:
                     # Trim leading and trailing whitespace
                     description = description.strip()
                     if description:
-                        # Fix any UTF-8/Windows-1252 mojibake in the description
-                        description = _fix_mojibake(description)
                         # Only add blockquote if description is not empty
                         # Replace newlines with newline + blockquote marker to maintain
                         # blockquote formatting
@@ -201,14 +162,18 @@ class FileWriter:
                         suffix = None
 
                     title = page_titles.get(docname, docname)
-                    # Fix any UTF-8/Windows-1252 mojibake in the title
+
-                    title = _fix_mojibake(title)
+                    # Avoid duplicate extensions when suffix matches
+                    # sourcelink_suffix (e.g., both are ".txt")
+                    effective_sourcelink_suffix = sourcelink_suffix
+                    if suffix and suffix == sourcelink_suffix:
+                        effective_sourcelink_suffix = ""
 
                     uri = uri_template.format(
                         base_url=base_url,
                         docname=docname,
                         suffix=suffix or "",
-                        sourcelink_suffix=sourcelink_suffix,
+                        sourcelink_suffix=effective_sourcelink_suffix,
                     )
 
                     f.write(f"- [{title}]({uri})\n")

tests/test_llms_txt.py

@@ -1182,48 +1182,3 @@ def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog):
     # Verify llms.txt was still created
     llms_txt = outdir / "llms.txt"
     assert llms_txt.exists()
-
-
-def test_fix_mojibake():
-    """
-    Test that the _fix_mojibake function correctly repairs UTF-8/Windows-1252 mojibake.
-    """
-    from sphinx_llms_txt.writer import _fix_mojibake
-
-    # Test case from issue #65: smart apostrophe corrupted
-    # U+2019 (') encoded as UTF-8 (E2 80 99) then decoded as Windows-1252 gives ’
-    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 99->™(U+2122)
-    corrupted = "What\u00e2\u20ac\u2122s New"  # ’
-    expected = "What\u2019s New"  # ' = U+2019
-    assert _fix_mojibake(corrupted) == expected
-
-    # Test left double quote: U+201C (") -> “
-    # U+201C encoded as UTF-8: E2 80 9C
-    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 9C->œ(U+0153)
-    corrupted_ldq = "He said \u00e2\u20ac\u0153Hello"
-    expected_ldq = "He said \u201cHello"
-    assert _fix_mojibake(corrupted_ldq) == expected_ldq
-
-    # Test em dash: U+2014 (—) -> â€"
-    # U+2014 encoded as UTF-8: E2 80 94
-    # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 94->"(U+201D)
-    corrupted_emdash = "one\u00e2\u20ac\u201dtwo"
-    expected_emdash = "one\u2014two"
-    assert _fix_mojibake(corrupted_emdash) == expected_emdash
-
-    # Test that already correct text is not modified
-    # Using Unicode escape for smart apostrophe
-    correct = "What\u2019s New In Our Latest Release!"
-    assert _fix_mojibake(correct) == correct
-
-    # Test empty string
-    assert _fix_mojibake("") == ""
-
-    # Test plain ASCII text passes through unchanged
-    plain = "Hello World"
-    assert _fix_mojibake(plain) == plain
-
-    # Test mixed mojibake and normal text
-    mixed = "Here\u00e2\u20ac\u2122s a test"
-    expected_mixed = "Here\u2019s a test"
-    assert _fix_mojibake(mixed) == expected_mixed

tests/test_uri_template.py

@@ -132,6 +132,48 @@ def test_uri_template_custom(tmp_path):
     assert "- [Home Page](https://example.com/raw/index.rst)" in content
 
 
+def test_uri_template_no_double_extension_when_suffix_matches_sourcelink(tmp_path):
+    """Test that .txt suffix + .txt sourcelink_suffix doesn't produce .txt.txt URLs."""
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+
+    sources_dir = build_dir / "_sources"
+    sources_dir.mkdir()
+
+    class MockApp:
+        class Config:
+            html_sourcelink_suffix = ".txt"
+
+        config = Config()
+
+    config = {
+        "llms_txt_file": True,
+        "llms_txt_filename": "llms.txt",
+        "html_baseurl": "https://example.com",
+    }
+    writer = FileWriter(config, str(build_dir), MockApp())
+
+    page_titles = {
+        "index": "Home Page",
+        "contents": "Table of Contents",
+    }
+
+    # .txt source files — suffix matches sourcelink_suffix
+    page_order = [("index", ".txt"), ("contents", ".txt")]
+
+    writer.write_verbose_info_to_file(page_order, page_titles, 0, sources_dir)
+
+    verbose_file = build_dir / "llms.txt"
+    with open(verbose_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # Should NOT have double .txt.txt extension
+    assert ".txt.txt" not in content
+    # Should have single .txt extension
+    assert "- [Home Page](https://example.com/_sources/index.txt)" in content
+    assert "- [Table of Contents](https://example.com/_sources/contents.txt)" in content
+
+
 def test_uri_template_invalid_fallback(tmp_path):
     """
     Test that invalid template falls back to default sources template when