diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 83059ae..5e3bd2e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Changelog ========= +0.7.2 +----- + +- Fix encoding of Unicode characters (smart quotes, em dashes, etc.) in llms.txt output + `#65 `_ + 0.7.1 ----- diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py index c64762b..def4ad5 100644 --- a/sphinx_llms_txt/__init__.py +++ b/sphinx_llms_txt/__init__.py @@ -21,7 +21,7 @@ from .manager import LLMSFullManager from .processor import DocumentProcessor from .writer import FileWriter -__version__ = "0.7.1" +__version__ = "0.7.2" # Export classes needed by tests __all__ = [ diff --git a/sphinx_llms_txt/writer.py b/sphinx_llms_txt/writer.py index 44b9279..73f19b1 100644 --- a/sphinx_llms_txt/writer.py +++ b/sphinx_llms_txt/writer.py @@ -11,6 +11,41 @@ from sphinx.util import logging logger = logging.getLogger(__name__) +def _fix_mojibake(text: str) -> str: + """Fix common UTF-8/Windows-1252 mojibake in text. + + Mojibake (文字化け, "character transformation") is a Japanese term for garbled text + caused by decoding bytes with the wrong character encoding. + + This handles the case where UTF-8 bytes were incorrectly decoded as Windows-1252 + (or Latin-1), resulting in corrupted characters like: + - ' (U+2019) becoming ’ + - " (U+201C) becoming “ + - " (U+201D) becoming †+ - — (U+2014) becoming â€" + - – (U+2013) becoming â€" + - … (U+2026) becoming … + + Args: + text: The potentially corrupted text string + + Returns: + The repaired text string, or the original if no repair was needed/possible + """ + if not text: + return text + + try: + # Try to encode the text as Windows-1252 (which would succeed if it contains + # the mojibake characters) and then decode as UTF-8 (to get the original) + return text.encode("windows-1252").decode("utf-8") + except (UnicodeDecodeError, UnicodeEncodeError): + # If encoding/decoding fails, the text is either: + # - Already correct UTF-8 + # - Corrupted in a different way we can't fix + return text + + class FileWriter: """Handles writing processed content to output files.""" @@ -119,6 +154,8 @@ class FileWriter: and hasattr(self.app.config, "project") ): project_name = self.app.config.project + # Fix any UTF-8/Windows-1252 mojibake in the project name + project_name = _fix_mojibake(project_name) f.write(f"# {project_name}\n\n") # Add description if available @@ -127,6 +164,8 @@ class FileWriter: # Trim leading and trailing whitespace description = description.strip() if description: + # Fix any UTF-8/Windows-1252 mojibake in the description + description = _fix_mojibake(description) # Only add blockquote if description is not empty # Replace newlines with newline + blockquote marker to maintain # blockquote formatting @@ -162,6 +201,8 @@ class FileWriter: suffix = None title = page_titles.get(docname, docname) + # Fix any UTF-8/Windows-1252 mojibake in the title + title = _fix_mojibake(title) uri = uri_template.format( base_url=base_url, diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py index 789b358..6bc840e 100644 --- a/tests/test_llms_txt.py +++ b/tests/test_llms_txt.py @@ -1182,3 +1182,48 @@ def test_llms_txt_no_warning_when_full_file_disabled(tmp_path, caplog): # Verify llms.txt was still created llms_txt = outdir / "llms.txt" assert llms_txt.exists() + + +def test_fix_mojibake(): + """ + Test that the _fix_mojibake function correctly repairs UTF-8/Windows-1252 mojibake. + """ + from sphinx_llms_txt.writer import _fix_mojibake + + # Test case from issue #65: smart apostrophe corrupted + # U+2019 (') encoded as UTF-8 (E2 80 99) then decoded as Windows-1252 gives ’ + # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 99->™(U+2122) + corrupted = "What\u00e2\u20ac\u2122s New" # ’ + expected = "What\u2019s New" # ' = U+2019 + assert _fix_mojibake(corrupted) == expected + + # Test left double quote: U+201C (") -> “ + # U+201C encoded as UTF-8: E2 80 9C + # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 9C->œ(U+0153) + corrupted_ldq = "He said \u00e2\u20ac\u0153Hello" + expected_ldq = "He said \u201cHello" + assert _fix_mojibake(corrupted_ldq) == expected_ldq + + # Test em dash: U+2014 (—) -> â€" + # U+2014 encoded as UTF-8: E2 80 94 + # In Windows-1252: E2->â(U+00E2), 80->€(U+20AC), 94->"(U+201D) + corrupted_emdash = "one\u00e2\u20ac\u201dtwo" + expected_emdash = "one\u2014two" + assert _fix_mojibake(corrupted_emdash) == expected_emdash + + # Test that already correct text is not modified + # Using Unicode escape for smart apostrophe + correct = "What\u2019s New In Our Latest Release!" + assert _fix_mojibake(correct) == correct + + # Test empty string + assert _fix_mojibake("") == "" + + # Test plain ASCII text passes through unchanged + plain = "Hello World" + assert _fix_mojibake(plain) == plain + + # Test mixed mojibake and normal text + mixed = "Here\u00e2\u20ac\u2122s a test" + expected_mixed = "Here\u2019s a test" + assert _fix_mojibake(mixed) == expected_mixed