diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 74da444..ea5c0f5 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,6 +1,11 @@
 Changelog
 =========
 
+0.5.1
+-----
+
+- Only allow builders that have _sources directory
+
 0.5.0
 -----
 
diff --git a/sphinx_llms_txt/__init__.py b/sphinx_llms_txt/__init__.py
index ca56d77..af96cf1 100644
--- a/sphinx_llms_txt/__init__.py
+++ b/sphinx_llms_txt/__init__.py
@@ -21,7 +21,7 @@ from .manager import LLMSFullManager
 from .processor import DocumentProcessor
 from .writer import FileWriter
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 
 # Export classes needed by tests
 __all__ = [
@@ -113,7 +113,6 @@ def build_finished(app: Sphinx, exception):
 def setup(app: Sphinx) -> Dict[str, Any]:
     """Set up the Sphinx extension."""
 
-    # Add configuration options
     app.add_config_value("llms_txt_file", True, "env")
     app.add_config_value("llms_txt_filename", "llms.txt", "env")
     app.add_config_value("llms_txt_full_file", True, "env")
@@ -127,14 +126,20 @@ def setup(app: Sphinx) -> Dict[str, Any]:
     app.add_config_value("llms_txt_code_files", [], "env")
     app.add_config_value("llms_txt_code_base_path", None, "env")
 
-    # Connect to Sphinx events
-    app.connect("doctree-resolved", doctree_resolved)
-    app.connect("build-finished", build_finished)
+    def builder_inited(app):
+        """Used to limit what builders are allowed to run the extension."""
 
-    # Reset manager and root paragraph for each build
-    global _manager, _root_first_paragraph
-    _manager = LLMSFullManager()
-    _root_first_paragraph = ""
+        allowed_builders = ["html", "singlehtml", "dirhtml"]
+        if hasattr(app, "builder") and app.builder.name in allowed_builders:
+            # Reset manager and root paragraph for each build
+            global _manager, _root_first_paragraph
+            _manager = LLMSFullManager()
+            _root_first_paragraph = ""
+
+            app.connect("doctree-resolved", doctree_resolved)
+            app.connect("build-finished", build_finished)
+
+    app.connect("builder-inited", builder_inited)
 
     return {
         "version": __version__,
diff --git a/tests/test_llms_txt.py b/tests/test_llms_txt.py
index 559997e..1ddf856 100644
--- a/tests/test_llms_txt.py
+++ b/tests/test_llms_txt.py
@@ -41,6 +41,42 @@ def test_setup_returns_valid_dict():
     assert "parallel_write_safe" in result
 
 
+def test_builder_inited_with_disallowed_builder():
+    """Test that disallowed builders do not trigger extension setup."""
+    import sphinx_llms_txt
+
+    # Reset global state
+    sphinx_llms_txt._manager = sphinx_llms_txt.LLMSFullManager()
+    sphinx_llms_txt._root_first_paragraph = ""
+
+    # Mock a Sphinx app with a disallowed builder
+    class MockBuilder:
+        name = "text"  # Not in allowed list
+
+    class MockApp:
+        def __init__(self):
+            self.config_values = {}
+            self.connections = {}
+            self.builder = MockBuilder()
+
+        def add_config_value(self, name, default, rebuild):
+            self.config_values[name] = (default, rebuild)
+
+        def connect(self, event, handler):
+            self.connections[event] = handler
+
+    app = MockApp()
+    setup(app)
+
+    # Trigger builder-inited
+    builder_inited_handler = app.connections["builder-inited"]
+    builder_inited_handler(app)
+
+    # With disallowed builder, other events should NOT be connected
+    assert "doctree-resolved" not in app.connections
+    assert "build-finished" not in app.connections
+
+
 def test_document_collector_initialization():
     """Test initialization of DocumentCollector."""
     collector = DocumentCollector()