test: 补充知识处理、检索与 Agent 回归用例

覆盖切块、向量检索、工具调用和历史预算；模型相关用例使用桩。

backend/tests/knowledge/test_chunking.py

@@ -0,0 +1,157 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+from agno.document import Document
+
+from app.knowledge.chunking import (
+    ChunkDocument,
+    PageChunking,
+    chunk_id,
+    chunk_page,
+    page_id_of,
+    page_title,
+    split_text,
+)
+
+
+class TestSplitText:
+    def test_short_text_is_one_piece(self):
+        assert split_text("hello world", size=100) == ["hello world"]
+
+    def test_pieces_respect_size_and_join_back_exactly(self):
+        text = "\n\n".join(
+            f"Paragraph {i}. " + " ".join(f"word{j}" for j in range(40)) for i in range(30)
+        )
+        pieces = split_text(text, size=300)
+        assert len(pieces) > 5
+        assert all(len(p) <= 300 for p in pieces)
+        assert "".join(pieces) == text
+
+    def test_whitespace_and_structure_preserved(self):
+        table = "| plan | price |\n|------|-------|\n| Free | $0    |\n| Pro  | $29   |\n"
+        text = (table + "\n") * 40
+        pieces = split_text(text, size=200)
+        assert "".join(pieces) == text
+        assert all("|" in p for p in pieces)
+
+    def test_prefers_paragraph_breaks(self):
+        text = "a" * 150 + "\n\n" + "b" * 150 + "\n\n" + "c" * 150
+        assert split_text(text, size=200) == ["a" * 150 + "\n\n", "b" * 150 + "\n\n", "c" * 150]
+
+    def test_hard_cut_when_no_separator(self):
+        pieces = split_text("x" * 1000, size=300)
+        assert [len(p) for p in pieces] == [300, 300, 300, 100]
+
+    @pytest.mark.parametrize("size", [1, 2, 3, 5, 10])
+    def test_always_terminates_and_covers_input_for_tiny_sizes(self, size):
+        text = "ab. cd\n\nef gh. ij kl\nmn op"
+        pieces = split_text(text, size=size)
+        assert "".join(pieces) == text
+        assert all(0 < len(p) <= size for p in pieces)
+
+    def test_rejects_zero_size(self):
+        with pytest.raises(ValueError):
+            split_text("abc", size=0)
+
+
+class TestIds:
+    def test_chunk_id_round_trip(self):
+        assert page_id_of(chunk_id("https://x.com/docs", 3)) == "https://x.com/docs"
+
+    def test_legacy_pdf_suffix_still_grouped(self):
+        assert page_id_of("manual.pdf_12") == "manual.pdf"
+
+    def test_url_ending_in_number_is_its_own_page(self):
+        # The `::` separator is why: `_2` alone would merge it into `/post`.
+        assert page_id_of("https://x.com/post_2::1") == "https://x.com/post_2"
+
+    def test_plain_id_unchanged(self):
+        assert page_id_of("getting_started") == "getting_started"
+
+
+class TestChunkPage:
+    def _page(self, content: str, **meta) -> Document:
+        return Document(id="https://x.com/pricing", name="https://x.com", content=content, meta_data=meta)
+
+    def test_small_page_returned_unchanged(self):
+        page = self._page("short", url="https://x.com/pricing")
+        assert chunk_page(page, size=100) == [page]
+
+    def test_large_page_becomes_numbered_disjoint_chunks_with_metadata(self):
+        body = "\n\n".join(f"Section {i}: " + "detail " * 30 for i in range(20))
+        page = self._page(body, url="https://x.com/pricing", agent_id=["a1"])
+        chunks = chunk_page(page, size=400, overlap=50)
+        assert len(chunks) > 1
+        assert [c.id for c in chunks] == [f"https://x.com/pricing::{i}" for i in range(1, len(chunks) + 1)]
+        # Stored text is disjoint: rejoining the page never duplicates anything.
+        assert "".join(c.content for c in chunks) == body
+        for i, c in enumerate(chunks, start=1):
+            assert c.name == "https://x.com"
+            assert c.meta_data["url"] == "https://x.com/pricing"
+            assert c.meta_data["agent_id"] == ["a1"]
+            assert c.meta_data["page_id"] == "https://x.com/pricing"
+            assert c.meta_data["chunk"] == i
+            assert c.meta_data["chunk_count"] == len(chunks)
+            assert c.meta_data["chunk_size"] == len(c.content)
+
+    def test_embedding_text_carries_title_and_previous_tail_but_content_does_not(self):
+        body = "Pricing plans\n" + "x " * 600
+        chunks = chunk_page(self._page(body), size=300, overlap=40)
+        embedder = MagicMock()
+        embedder.get_embedding_and_usage.return_value = ([0.1], None)
+        chunks[1].embed(embedder=embedder)
+        embedded_text = embedder.get_embedding_and_usage.call_args.args[0]
+        assert embedded_text == "Pricing plans\n" + chunks[0].content[-40:] + chunks[1].content
+        assert "Pricing plans" not in chunks[1].content
+        # A first line guessed from the content is for the embedding only; a
+        # crawled first line is as likely to be a cookie banner as a heading.
+        assert "title" not in chunks[1].meta_data
+
+    def test_zero_overlap_embeds_title_and_chunk_only(self):
+        chunks = chunk_page(self._page("T\n" + "y " * 400), size=200, overlap=0)
+        assert chunks[1].embedding_text == "T\n" + chunks[1].content
+
+    def test_explicit_title_wins(self):
+        page = self._page("first line\n" + "y " * 400, title="Given Title")
+        assert page_title(page) == "Given Title"
+        assert chunk_page(page, size=200)[0].meta_data["title"] == "Given Title"
+
+    def test_stale_chunk_metadata_is_not_carried_over(self):
+        page = self._page("w " * 400, url="u", chunk=1, chunk_count=5, chunk_size=9, page_id="old")
+        chunks = chunk_page(page, size=200)
+        assert chunks[0].meta_data["chunk_count"] == len(chunks)
+        assert chunks[0].meta_data["page_id"] == "https://x.com/pricing"
+
+    def test_small_page_whose_id_ends_in_number_gets_explicit_chunk_id(self):
+        page = Document(id="https://x.com/post_2", name="https://x.com", content="short", meta_data={"url": "u"})
+        chunks = chunk_page(page, size=100)
+        assert [c.id for c in chunks] == ["https://x.com/post_2::1"]
+        assert chunks[0].content == "short"
+        assert chunks[0].meta_data["chunk_count"] == 1
+        assert page_id_of(chunks[0].id) == "https://x.com/post_2"
+
+    def test_chunk_document_embed_requires_embedder(self):
+        with pytest.raises(ValueError):
+            ChunkDocument(content="x").embed()
+
+    def test_strategy_adapter_uses_same_splitter(self):
+        page = self._page("z " * 500)
+        assert [c.id for c in PageChunking(size=200).chunk(page)] == [
+            c.id for c in chunk_page(page, size=200)
+        ]

backend/tests/knowledge/test_content_summarizer.py

@@ -0,0 +1,379 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import pytest
+from unittest.mock import patch, MagicMock
+from app.knowledge.content_summarizer import ContentSummarizer, get_content_summarizer
+
+
+class TestContentSummarizer:
+    """Test suite for ContentSummarizer class"""
+
+    def test_init_disabled(self):
+        """Test initialization when summarization is disabled"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = False
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = ""
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            summarizer = ContentSummarizer()
+
+            assert not summarizer.enabled
+            assert summarizer._agent is None
+
+    def test_init_enabled_no_api_key(self):
+        """Test initialization when enabled but no API key"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = ""
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            summarizer = ContentSummarizer()
+
+            assert not summarizer.enabled
+
+    def test_init_enabled_with_api_key(self):
+        """Test initialization when enabled with API key"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            summarizer = ContentSummarizer()
+
+            assert summarizer.enabled
+            assert summarizer.api_key == "test_api_key"
+
+    def test_init_openai_compatible_no_base_url(self):
+        """OPENAI_COMPATIBLE without a base_url disables summarization, like a missing API key"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "OPENAI_COMPATIBLE"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "local-model"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.KNOWLEDGE_SUMMARY_BASE_URL = ""
+
+            summarizer = ContentSummarizer()
+
+            assert not summarizer.enabled
+
+    def test_init_openai_compatible_with_base_url(self):
+        """OPENAI_COMPATIBLE with a base_url stays enabled and passes base_url to create_model"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings, \
+             patch('app.knowledge.content_summarizer.create_model') as mock_create_model:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "OPENAI_COMPATIBLE"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "local-model"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.KNOWLEDGE_SUMMARY_BASE_URL = "http://localhost:11434/v1"
+
+            summarizer = ContentSummarizer()
+            assert summarizer.enabled
+
+            summarizer._get_agent()
+
+            mock_create_model.assert_called_once_with(
+                model_type="OPENAI_COMPATIBLE",
+                api_key="test_api_key",
+                model_name="local-model",
+                max_tokens=4000,
+                base_url="http://localhost:11434/v1"
+            )
+
+    def test_summarize_disabled(self):
+        """Test that original content is returned when summarization is disabled"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = False
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = ""
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            summarizer = ContentSummarizer()
+
+            content = "This is test content with a URL: https://example.com/product/123"
+            result = summarizer.summarize(content, "https://example.com")
+
+            assert result == content
+
+    def test_summarize_short_content(self):
+        """Test that short content is not summarized"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            summarizer = ContentSummarizer()
+
+            # Content under 1000 chars
+            content = "Short content with URL: https://example.com/product/123"
+            result = summarizer.summarize(content, "https://example.com")
+
+            assert result == content
+
+    def test_summarize_preserves_urls(self):
+        """Test that URLs are preserved in summarization"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings, \
+             patch('app.knowledge.content_summarizer.create_model') as mock_create_model:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.ENVIRONMENT = "development"
+
+            # Mock the model
+            mock_model = MagicMock()
+            mock_create_model.return_value = mock_model
+
+            # Create mock response with URLs preserved
+            mock_response = MagicMock()
+            mock_response.content = "Summary of the product. Check out the product at https://example.com/product/123 and related items at https://example.com/category/products"
+
+            # Mock the agent and its run method
+            mock_agent = MagicMock()
+            mock_agent.run.return_value = mock_response
+
+            summarizer = ContentSummarizer()
+
+            # Override the agent with our mock
+            with patch('app.knowledge.content_summarizer.Agent', return_value=mock_agent):
+                # Get the agent to initialize it
+                summarizer._get_agent()
+
+                # Long content with multiple URLs
+                content = """
+                This is a long piece of content about our products.
+                We have many great products available at https://example.com/product/123
+                and you can find more information at https://example.com/category/products.
+                Our documentation is available at https://docs.example.com/guide
+                and images can be found at https://cdn.example.com/images/product.jpg.
+                """ * 50  # Make it long enough to trigger summarization
+
+                result = summarizer.summarize(content, "https://example.com")
+
+                # Verify URLs are in the result
+                assert "https://example.com/product/123" in result
+                assert "https://example.com/category/products" in result
+
+    def test_summarize_adds_source_url(self):
+        """Test that source URL is added if not present"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings, \
+             patch('app.knowledge.content_summarizer.create_model') as mock_create_model:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.ENVIRONMENT = "development"
+
+            # Mock the model
+            mock_model = MagicMock()
+            mock_create_model.return_value = mock_model
+
+            # Create mock response without source URL
+            mock_response = MagicMock()
+            mock_response.content = "Summary of the product without source URL"
+
+            # Mock the agent and its run method
+            mock_agent = MagicMock()
+            mock_agent.run.return_value = mock_response
+
+            summarizer = ContentSummarizer()
+
+            # Override the agent with our mock
+            with patch('app.knowledge.content_summarizer.Agent', return_value=mock_agent):
+                # Get the agent to initialize it
+                summarizer._get_agent()
+
+                # Long content
+                content = "This is a long piece of content about our products. " * 100
+
+                result = summarizer.summarize(content, "https://example.com/products")
+
+                # Verify source URL is added
+                assert "Source: https://example.com/products" in result
+
+    def test_summarize_error_handling(self):
+        """Test that errors are handled gracefully and original content is returned"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings, \
+             patch('app.knowledge.content_summarizer.create_model') as mock_create_model:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.ENVIRONMENT = "development"
+
+            # Mock the model to raise an error
+            mock_create_model.side_effect = Exception("API Error")
+
+            summarizer = ContentSummarizer()
+
+            # Long content
+            content = "This is a long piece of content with URL: https://example.com/product/123 " * 100
+
+            result = summarizer.summarize(content, "https://example.com")
+
+            # Verify original content is returned on error
+            assert result == content
+
+    def test_get_content_summarizer_singleton(self):
+        """Test that get_content_summarizer returns a singleton instance"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+
+            # Reset the global instance
+            import app.knowledge.content_summarizer
+            app.knowledge.content_summarizer._summarizer = None
+
+            # Get the instance twice
+            instance1 = get_content_summarizer()
+            instance2 = get_content_summarizer()
+
+            # Verify it's the same instance
+            assert instance1 is instance2
+
+    def test_summarizer_instructions_include_url_preservation(self):
+        """Test that the summarizer instructions emphasize URL preservation"""
+        with patch('app.knowledge.content_summarizer.settings') as mock_settings, \
+             patch('app.knowledge.content_summarizer.create_model') as mock_create_model, \
+             patch('app.knowledge.content_summarizer.Agent') as mock_agent_class:
+            mock_settings.KNOWLEDGE_SUMMARY_ENABLED = True
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_TYPE = "GROQ"
+            mock_settings.KNOWLEDGE_SUMMARY_MODEL_NAME = "llama-3.1-8b-instant"
+            mock_settings.KNOWLEDGE_SUMMARY_API_KEY = "test_api_key"
+            mock_settings.KNOWLEDGE_SUMMARY_MAX_TOKENS = 4000
+            mock_settings.ENVIRONMENT = "development"
+
+            # Mock the model
+            mock_model = MagicMock()
+            mock_create_model.return_value = mock_model
+
+            # Mock the agent
+            mock_agent_instance = MagicMock()
+            mock_agent_class.return_value = mock_agent_instance
+
+            summarizer = ContentSummarizer()
+            summarizer._get_agent()
+
+            # Verify Agent was called with instructions containing URL preservation
+            assert mock_agent_class.called
+            call_kwargs = mock_agent_class.call_args.kwargs
+            instructions = call_kwargs['instructions']
+
+            # Check that instructions emphasize URL preservation
+            assert 'PRESERVING ALL URLs' in instructions
+            assert 'Product URLs' in instructions
+            assert 'DO NOT remove, shorten, or modify ANY URLs' in instructions
+
+
+class TestChatAgentURLInstructions:
+    """Test suite for verifying chat agent includes URL instructions"""
+
+    def test_chat_agent_includes_url_instructions(self):
+        """Test that chat agent instructions include URL preservation"""
+        from app.agents.chat_agent import ChatAgent
+        from uuid import uuid4
+
+        # Mock all database and external dependencies
+        with patch('app.agents.chat_agent.SessionLocal') as mock_session_local, \
+             patch('app.tools.knowledge_search_byagent.SessionLocal') as mock_knowledge_session_local, \
+             patch('app.agents.chat_agent.JiraRepository') as mock_jira_repo, \
+             patch('app.agents.chat_agent.AgentShopifyConfigRepository') as mock_shopify_repo, \
+             patch('app.repositories.lead_capture.LeadCaptureConfigRepository') as mock_lead_repo, \
+             patch('app.agents.chat_agent.EncryptedPostgresAgentStorage') as mock_storage, \
+             patch('app.agents.chat_agent.create_model') as mock_create_model, \
+             patch('app.agents.chat_agent.Agent') as mock_agent_class:
+
+            # No lead-capture config for this agent (returns None → capture off).
+            mock_lead_repo.return_value.get_by_agent.return_value = None
+
+            # Mock database session
+            mock_db = MagicMock()
+            mock_session_local.return_value.__enter__.return_value = mock_db
+            mock_session_local.return_value.__exit__.return_value = None
+
+            # Mock knowledge search database session
+            mock_knowledge_session_local.return_value.__enter__.return_value = mock_db
+            mock_knowledge_session_local.return_value.__exit__.return_value = None
+
+            # Mock Jira repository
+            from app.models.schemas.jira import AgentWithJiraConfig
+            from app.models.agent import AgentType
+
+            mock_agent_data = AgentWithJiraConfig(
+                id=uuid4(),
+                name="Test Agent",
+                display_name="Test Agent",
+                description="Test",
+                instructions=["You are a helpful assistant"],
+                tools=[],
+                agent_type=AgentType.CUSTOMER_SUPPORT,
+                is_default=False,
+                is_active=True,
+                organization_id=uuid4(),
+                transfer_to_human=False,
+                ask_for_rating=False,
+                knowledge=[],
+                jira_enabled=False,
+                jira_project_key=None,
+                jira_issue_type_id=None,
+                groups=[],
+                organization=None
+            )
+
+            mock_jira_repo.return_value.get_agent_with_jira_config.return_value = mock_agent_data
+            mock_shopify_repo.return_value.get_agent_shopify_config.return_value = None
+
+            # Mock storage and model
+            mock_storage.return_value = MagicMock()
+            mock_create_model.return_value = MagicMock()
+            mock_agent_instance = MagicMock()
+            mock_agent_class.return_value = mock_agent_instance
+
+            # Create chat agent
+            chat_agent = ChatAgent(
+                api_key="test_key",
+                model_name="gpt-4",
+                model_type="OPENAI",
+                org_id=str(uuid4()),
+                agent_id=str(mock_agent_data.id),
+                session_id=str(uuid4())
+            )
+
+            # Verify Agent was called with instructions containing URL guidance
+            assert mock_agent_class.called
+            call_kwargs = mock_agent_class.call_args.kwargs
+            instructions = call_kwargs['instructions']
+
+            # Check that instructions include URL preservation guidance
+            assert 'Include URLs in Responses' in instructions or 'MUST include these URLs' in instructions
+            assert 'URLs' in instructions

backend/tests/knowledge/test_crawl4ai_fallback.py

@@ -0,0 +1,490 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import pytest
+from unittest.mock import MagicMock, patch, AsyncMock
+from bs4 import BeautifulSoup
+import asyncio
+
+from app.knowledge.crawl4ai_fallback import (
+    Crawl4AIFallback,
+    get_crawl4ai_fallback,
+    CRAWL4AI_AVAILABLE
+)
+
+
+# Live integration tests below hit the real network and launch a real Chromium
+# browser. On CI (Python 3.12) the event-loop teardown of fetch_with_browser can
+# raise "Event loop is closed" during GC and leave a coroutine un-awaited, which
+# then fails an unrelated test. Skip them in CI by default; run them locally with
+# RUN_LIVE_CRAWL_TESTS=1 to exercise the real path.
+_SKIP_LIVE = os.getenv("CI") is not None and os.getenv("RUN_LIVE_CRAWL_TESTS") != "1"
+_LIVE_REASON = "live crawl4ai network/browser test (set RUN_LIVE_CRAWL_TESTS=1 to run; skipped in CI)"
+
+
+class TestCrawl4AIFallback:
+    """Test suite for Crawl4AIFallback class"""
+    
+    def test_init_default_params(self):
+        """Test initialization with default parameters"""
+        fallback = Crawl4AIFallback()
+        
+        assert fallback.timeout == 30
+        assert fallback.verify_ssl is True
+        assert fallback._is_available == CRAWL4AI_AVAILABLE
+    
+    def test_init_custom_params(self):
+        """Test initialization with custom parameters"""
+        fallback = Crawl4AIFallback(timeout=60, verify_ssl=False)
+        
+        assert fallback.timeout == 60
+        assert fallback.verify_ssl is False
+    
+    def test_is_available_property(self):
+        """Test is_available property"""
+        fallback = Crawl4AIFallback()
+        
+        assert fallback.is_available == CRAWL4AI_AVAILABLE
+    
+    @pytest.mark.asyncio
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
+    async def test_async_fetch_success(self, mock_crawler_class):
+        """Test successful async fetch without screenshot"""
+        # Arrange
+        url = "https://example.com"
+        mock_result = MagicMock()
+        mock_result.success = True
+        mock_result.html = "<html><body>Test content</body></html>"
+        mock_result.markdown = MagicMock()
+        mock_result.markdown.raw_markdown = "# Test content"
+        mock_result.error_message = None
+        
+        mock_crawler_instance = AsyncMock()
+        mock_crawler_instance.arun.return_value = mock_result
+        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
+        mock_crawler_instance.__aexit__.return_value = None
+        mock_crawler_class.return_value = mock_crawler_instance
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
+        
+        # Assert
+        assert html == "<html><body>Test content</body></html>"
+        assert markdown == "# Test content"
+        assert screenshot is None
+        mock_crawler_instance.arun.assert_called_once()
+    
+    @pytest.mark.asyncio
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
+    async def test_async_fetch_failure(self, mock_crawler_class):
+        """Test failed async fetch"""
+        # Arrange
+        url = "https://example.com"
+        mock_result = MagicMock()
+        mock_result.success = False
+        mock_result.error_message = "Connection timeout"
+        
+        mock_crawler_instance = AsyncMock()
+        mock_crawler_instance.arun.return_value = mock_result
+        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
+        mock_crawler_instance.__aexit__.return_value = None
+        mock_crawler_class.return_value = mock_crawler_instance
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
+        
+        # Assert
+        assert html is None
+        assert markdown is None
+        assert screenshot is None
+    
+    @pytest.mark.asyncio
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('playwright.async_api.async_playwright')
+    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
+    async def test_async_fetch_with_screenshot(self, mock_crawler_class, mock_playwright):
+        """Test async fetch with screenshot enabled"""
+        # Arrange
+        url = "https://example.com"
+        
+        # Mock Playwright screenshot
+        mock_page = AsyncMock()
+        mock_page.goto = AsyncMock()
+        mock_page.wait_for_timeout = AsyncMock()
+        mock_page.screenshot.return_value = b'fake_screenshot_bytes'
+        mock_page.content.return_value = "<html><body>Playwright content</body></html>"
+        
+        mock_context = AsyncMock()
+        mock_context.new_page.return_value = mock_page
+        
+        mock_browser = AsyncMock()
+        mock_browser.new_context.return_value = mock_context
+        mock_browser.close = AsyncMock()
+        
+        mock_playwright_instance = AsyncMock()
+        mock_playwright_instance.chromium.launch.return_value = mock_browser
+        mock_playwright_instance.__aenter__.return_value = mock_playwright_instance
+        mock_playwright_instance.__aexit__.return_value = None
+        mock_playwright.return_value = mock_playwright_instance
+        
+        # Mock Crawl4AI
+        mock_result = MagicMock()
+        mock_result.success = True
+        mock_result.html = "<html><body>Crawl4AI content</body></html>"
+        mock_result.markdown = MagicMock()
+        mock_result.markdown.raw_markdown = "# Crawl4AI content"
+        
+        mock_crawler_instance = AsyncMock()
+        mock_crawler_instance.arun.return_value = mock_result
+        mock_crawler_instance.__aenter__.return_value = mock_crawler_instance
+        mock_crawler_instance.__aexit__.return_value = None
+        mock_crawler_class.return_value = mock_crawler_instance
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=True)
+        
+        # Assert
+        assert html == "<html><body>Playwright content</body></html>"  # Playwright HTML is used
+        assert markdown == "# Crawl4AI content"
+        assert screenshot is not None
+        assert isinstance(screenshot, str)  # Base64 encoded
+        mock_page.screenshot.assert_called_once()
+    
+    @pytest.mark.asyncio
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
+    async def test_async_fetch_exception(self, mock_crawler_class):
+        """Test async fetch with exception"""
+        # Arrange
+        url = "https://example.com"
+        mock_crawler_class.side_effect = Exception("Network error")
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
+        
+        # Assert
+        assert html is None
+        assert markdown is None
+        assert screenshot is None
+    
+    @pytest.mark.asyncio
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('app.knowledge.crawl4ai_fallback.AsyncWebCrawler')
+    async def test_async_fetch_blocks_internal_hosts(self, mock_crawler_class):
+        """The browser fallback must refuse internal/link-local targets. Without this
+        guard it is an SSRF primitive: a real Chromium fetching cloud metadata and
+        internal services, bypassing the httpx path's url_safety checks."""
+        from app.knowledge.url_safety import BlockedHostError
+
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+
+        for url in ("http://169.254.169.254/latest/meta-data/",
+                    "http://127.0.0.1:8000/internal",
+                    "http://10.0.0.5/admin"):
+            with pytest.raises(BlockedHostError):
+                await fallback._async_fetch(url, take_screenshot=False)
+
+        mock_crawler_class.assert_not_called()
+
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', False)
+    def test_fetch_with_browser_not_available(self):
+        """Test fetch_with_browser when Crawl4AI is not available"""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = False
+        
+        content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
+        
+        assert content is None
+        assert soup is None
+        assert screenshot is None
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_fetch_with_browser_with_markdown(self):
+        """Test fetch_with_browser returns markdown when available"""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Mock _run_async_safely to return markdown
+        html_content = "<html><body>Test</body></html>"
+        markdown_content = "# Test Markdown\n\nThis is a long markdown content with more than 100 characters to ensure it's substantial enough."
+        
+        with patch.object(fallback, '_run_async_safely', return_value=(html_content, markdown_content, None)):
+            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
+            
+            assert content == markdown_content.strip()
+            assert isinstance(soup, BeautifulSoup)
+            assert screenshot is None
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_fetch_with_browser_without_markdown(self):
+        """Test fetch_with_browser when markdown is not substantial"""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Mock _run_async_safely to return short markdown
+        html_content = "<html><body>Test</body></html>"
+        markdown_content = "Short"  # Less than 100 chars
+        
+        with patch.object(fallback, '_run_async_safely', return_value=(html_content, markdown_content, None)):
+            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
+            
+            assert content is None  # Should return None for extraction from soup
+            assert isinstance(soup, BeautifulSoup)
+            assert screenshot is None
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_fetch_with_browser_no_content(self):
+        """Test fetch_with_browser when no content is returned"""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        with patch.object(fallback, '_run_async_safely', return_value=(None, None, None)):
+            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
+            
+            assert content is None
+            assert soup is None
+            assert screenshot is None
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_fetch_with_browser_exception(self):
+        """Test fetch_with_browser handles exceptions gracefully"""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        with patch.object(fallback, '_run_async_safely', side_effect=Exception("Test error")):
+            content, soup, screenshot = fallback.fetch_with_browser("https://example.com")
+            
+            assert content is None
+            assert soup is None
+            assert screenshot is None
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('asyncio.get_event_loop')
+    @patch('asyncio.run')
+    def test_run_async_safely_no_running_loop(self, mock_asyncio_run, mock_get_loop):
+        """Test _run_async_safely when no event loop is running"""
+        # Arrange
+        mock_loop = MagicMock()
+        mock_loop.is_running.return_value = False
+        mock_get_loop.return_value = mock_loop
+        
+        mock_asyncio_run.return_value = ("<html>Test</html>", "# Test", None)
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)
+        
+        # Assert
+        assert html == "<html>Test</html>"
+        assert markdown == "# Test"
+        assert screenshot is None
+        mock_asyncio_run.assert_called_once()
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('asyncio.get_event_loop')
+    @patch('threading.Thread')
+    def test_run_async_safely_with_running_loop(self, mock_thread_class, mock_get_loop):
+        """Test _run_async_safely when event loop is already running"""
+        # Arrange
+        mock_loop = MagicMock()
+        mock_loop.is_running.return_value = True
+        mock_get_loop.return_value = mock_loop
+        
+        # Mock thread execution
+        mock_thread = MagicMock()
+        mock_thread.is_alive.return_value = False
+        mock_thread_class.return_value = mock_thread
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Mock the async fetch to set result
+        def mock_thread_target(*args, **kwargs):
+            # Simulate successful execution
+            pass
+        
+        with patch.object(fallback, '_async_fetch', return_value=("<html>Test</html>", "# Test", None)):
+            # Act
+            html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)
+            
+            # Assert
+            mock_thread_class.assert_called_once()
+            mock_thread.start.assert_called_once()
+            mock_thread.join.assert_called_once()
+    
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    @patch('asyncio.get_event_loop')
+    @patch('asyncio.run')
+    def test_run_async_safely_exception(self, mock_asyncio_run, mock_get_loop):
+        """Test _run_async_safely handles exceptions"""
+        # Arrange
+        mock_loop = MagicMock()
+        mock_loop.is_running.return_value = False
+        mock_get_loop.return_value = mock_loop
+        
+        # Simulate exception in asyncio.run
+        mock_asyncio_run.side_effect = RuntimeError("Async execution failed")
+        
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+        
+        # Act
+        html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)
+        
+        # Assert - should handle exception gracefully and return None
+        assert html is None
+        assert markdown is None
+        assert screenshot is None
+
+
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_run_async_safely_threaded_execution(self):
+        """Cover the real new-thread/new-loop path when an event loop is already running.
+
+        Mocks _async_fetch so no real network/browser is touched, but lets the actual
+        threading + new event loop machinery run (the path the live test used to cover).
+        """
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+
+        expected = ("<html><body>Threaded</body></html>", "# Threaded content", None)
+
+        fake_loop = MagicMock()
+        fake_loop.is_running.return_value = True
+
+        with patch('asyncio.get_event_loop', return_value=fake_loop), \
+             patch.object(fallback, '_async_fetch', new=AsyncMock(return_value=expected)):
+            html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)
+
+        assert (html, markdown, screenshot) == expected
+
+    @patch('app.knowledge.crawl4ai_fallback.CRAWL4AI_AVAILABLE', True)
+    def test_run_async_safely_threaded_exception(self):
+        """An exception inside the worker thread is re-raised and handled gracefully."""
+        fallback = Crawl4AIFallback()
+        fallback._is_available = True
+
+        fake_loop = MagicMock()
+        fake_loop.is_running.return_value = True
+
+        with patch('asyncio.get_event_loop', return_value=fake_loop), \
+             patch.object(fallback, '_async_fetch', new=AsyncMock(side_effect=RuntimeError("boom"))):
+            html, markdown, screenshot = fallback._run_async_safely("https://example.com", False)
+
+        assert html is None
+        assert markdown is None
+        assert screenshot is None
+
+
+class TestGetCrawl4AIFallback:
+    """Test suite for get_crawl4ai_fallback singleton function"""
+    
+    def test_get_crawl4ai_fallback_creates_instance(self):
+        """Test that get_crawl4ai_fallback creates an instance"""
+        # Reset singleton
+        import app.knowledge.crawl4ai_fallback as module
+        module._crawl4ai_fallback_instance = None
+        
+        instance = get_crawl4ai_fallback()
+        
+        assert isinstance(instance, Crawl4AIFallback)
+        assert instance.timeout == 30
+        assert instance.verify_ssl is True
+    
+    def test_get_crawl4ai_fallback_returns_same_instance(self):
+        """Test that get_crawl4ai_fallback returns the same instance (singleton)"""
+        # Reset singleton
+        import app.knowledge.crawl4ai_fallback as module
+        module._crawl4ai_fallback_instance = None
+        
+        instance1 = get_crawl4ai_fallback()
+        instance2 = get_crawl4ai_fallback()
+        
+        assert instance1 is instance2
+    
+    def test_get_crawl4ai_fallback_custom_params(self):
+        """Test get_crawl4ai_fallback with custom parameters"""
+        # Reset singleton
+        import app.knowledge.crawl4ai_fallback as module
+        module._crawl4ai_fallback_instance = None
+        
+        instance = get_crawl4ai_fallback(timeout=60, verify_ssl=False)
+        
+        assert instance.timeout == 60
+        assert instance.verify_ssl is False
+
+
+class TestCrawl4AIIntegration:
+    """Integration tests for Crawl4AI (only run if Crawl4AI is available)"""
+    
+    @pytest.mark.skipif(not CRAWL4AI_AVAILABLE or _SKIP_LIVE, reason=_LIVE_REASON)
+    @pytest.mark.asyncio
+    async def test_real_async_fetch_simple_page(self):
+        """Test real async fetch with a simple page (requires Crawl4AI)"""
+        fallback = Crawl4AIFallback(timeout=10)
+        
+        # Use a simple, reliable test URL
+        url = "https://example.com"
+        
+        try:
+            html, markdown, screenshot = await fallback._async_fetch(url, take_screenshot=False)
+            
+            # Basic assertions
+            assert html is not None or markdown is not None
+            if html:
+                assert len(html) > 0
+                assert "example" in html.lower()
+        except Exception as e:
+            pytest.skip(f"Network or Crawl4AI issue: {str(e)}")
+    
+    @pytest.mark.skipif(not CRAWL4AI_AVAILABLE or _SKIP_LIVE, reason=_LIVE_REASON)
+    def test_real_fetch_with_browser(self):
+        """Test real fetch_with_browser (requires Crawl4AI)"""
+        fallback = Crawl4AIFallback(timeout=10)
+        
+        if not fallback.is_available:
+            pytest.skip("Crawl4AI not available")
+        
+        url = "https://example.com"
+        
+        try:
+            content, soup, screenshot = fallback.fetch_with_browser(url, take_screenshot=False)
+            
+            # Basic assertions
+            assert content is not None or soup is not None
+            if soup:
+                assert isinstance(soup, BeautifulSoup)
+        except Exception as e:
+            pytest.skip(f"Network or Crawl4AI issue: {str(e)}")
+

backend/tests/knowledge/test_crawl_scope.py

@@ -0,0 +1,188 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import sys
+from pathlib import Path
+
+from app.knowledge.crawl_scope import (
+    DEFAULT_CRAWL_SCOPE,
+    SCOPE_DOMAIN,
+    SCOPE_HOST,
+    SCOPE_PATH,
+    CrawlScope,
+    is_crawlable_url,
+    url_scheme,
+)
+
+
+def test_url_scheme_ignores_ports():
+    assert url_scheme("https://site.com/a") == "https"
+    assert url_scheme("HTTP://site.com") == "http"
+    assert url_scheme("mailto:hi@site.com") == "mailto"
+    assert url_scheme("tel:+441234567890") == "tel"
+    assert url_scheme("javascript:void(0)") == "javascript"
+    # A scheme-less host:port must not read as a scheme (that would make
+    # _normalize_url leave it alone and the crawl skip a real page).
+    assert url_scheme("site.com:8443/x") == ""
+    assert url_scheme("/relative/path") == ""
+    assert url_scheme("") == ""
+
+
+def test_is_crawlable_url():
+    assert is_crawlable_url("https://site.com/a")
+    assert is_crawlable_url("/relative/path")  # resolved against a base later
+    assert not is_crawlable_url("mailto:hi@site.com")
+    assert not is_crawlable_url("tel:+441234567890")
+    assert not is_crawlable_url("sms:+441234567890")
+    assert not is_crawlable_url("javascript:void(0)")
+    assert not is_crawlable_url("data:text/html,<h1>x</h1>")
+
+
+def test_host_scope_is_the_default():
+    scope = CrawlScope.for_seed("https://help.site.com/hc/en")
+    assert scope.mode == DEFAULT_CRAWL_SCOPE == SCOPE_HOST
+    assert scope.allows("https://help.site.com/hc/en/articles/1")
+    # The rest of the company web presence stays out — the reported bug.
+    assert not scope.allows("https://www.site.com/all-integrations")
+    assert not scope.allows("https://site.com/blog/announcement")
+    assert not scope.allows("https://support.site.com/hc/help-center/en")
+    assert not scope.allows("https://other.com/a")
+
+
+def test_host_scope_treats_www_as_the_same_host():
+    scope = CrawlScope.for_seed("https://site.com/")
+    assert scope.allows("https://www.site.com/pricing")
+    assert scope.allows("http://SITE.com/pricing")
+    assert not scope.allows("https://blog.site.com/post")
+
+
+def test_canonical_puts_links_on_the_seeds_host_spelling():
+    """example.com/terms and www.example.com/terms are one page; storing both
+    spends two of the source's sub-page slots on the same text."""
+    bare = CrawlScope.for_seed("https://site.com/")
+    assert bare.canonical("https://www.site.com/terms") == "https://site.com/terms"
+    assert bare.canonical("https://site.com/terms") == "https://site.com/terms"
+
+    # Toward the SEED, not blindly www-stripped: a site may serve only the www
+    # form, and the seed is the spelling the crawl already fetched.
+    www = CrawlScope.for_seed("https://www.site.com/")
+    assert www.canonical("https://site.com/terms") == "https://www.site.com/terms"
+
+    # Port and path survive the rewrite.
+    assert bare.canonical("https://www.site.com:8443/a/b") == "https://site.com:8443/a/b"
+
+
+def test_canonical_leaves_other_hosts_alone():
+    scope = CrawlScope.for_seed("https://site.com/", SCOPE_DOMAIN)
+    # A real subdomain is a different site, even under domain scope.
+    assert scope.canonical("https://blog.site.com/post") == "https://blog.site.com/post"
+    assert scope.canonical("https://other.com/a") == "https://other.com/a"
+    # Nothing to rewrite, nothing to crash on.
+    assert scope.canonical("mailto:hi@site.com") == "mailto:hi@site.com"
+    assert scope.canonical("https://[bad") == "https://[bad"
+    assert scope.canonical("") == ""
+
+
+def test_domain_scope_keeps_the_previous_reach():
+    scope = CrawlScope.for_seed("https://help.site.com/hc/en", SCOPE_DOMAIN)
+    assert scope.allows("https://www.site.com/all-integrations")
+    assert scope.allows("https://support.site.com/x")
+    # Equality, not a suffix match: a lookalike domain must not slip through.
+    assert not scope.allows("https://evilsite.com/c")
+    assert not scope.allows("https://other.com/a")
+
+
+def test_path_scope_matches_on_segment_boundaries():
+    scope = CrawlScope.for_seed("https://site.com/hc/en", SCOPE_PATH)
+    assert scope.allows("https://site.com/hc/en")
+    assert scope.allows("https://site.com/hc/en/")
+    assert scope.allows("https://site.com/hc/en/articles/1")
+    assert not scope.allows("https://site.com/hc/entertainment")
+    assert not scope.allows("https://site.com/blog/post")
+    # Still host-bound.
+    assert not scope.allows("https://help.site.com/hc/en/articles/1")
+
+
+def test_path_scope_from_a_root_seed_behaves_like_host_scope():
+    scope = CrawlScope.for_seed("https://site.com/", SCOPE_PATH)
+    assert scope.allows("https://site.com/anything/deep")
+
+
+def test_no_scope_allows_non_http_urls():
+    """The mailto: crawl bug: 'https://' + 'mailto:a@site.com' parses as host
+    site.com with 'mailto:a' as userinfo, so a naive domain check passed it."""
+    for mode in (SCOPE_PATH, SCOPE_HOST, SCOPE_DOMAIN):
+        scope = CrawlScope.for_seed("https://site.com/", mode)
+        assert not scope.allows("mailto:hi@site.com")
+        assert not scope.allows("tel:+441234567890")
+        assert not scope.allows("https://mailto:hi@site.com/pay-by-bank")
+
+
+def test_unknown_mode_falls_back_to_the_default():
+    scope = CrawlScope.for_seed("https://site.com/", "everything")
+    assert scope.mode == DEFAULT_CRAWL_SCOPE
+
+
+def _pruner():
+    """The cleanup script lives outside the app tree (it must not import `app`),
+    so it re-implements the scope check. Load it the way it runs on a host."""
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts"))
+    try:
+        import prune_offsite_pages
+
+        return prune_offsite_pages
+    finally:
+        sys.path.pop(0)
+
+
+def test_pruner_scope_agrees_with_the_crawler():
+    """The script deletes what the crawler would no longer store, so its copy of
+    the rules must not drift from CrawlScope."""
+    pruner = _pruner()
+    seed = "https://help.site.com/hc/en"
+    urls = [
+        "https://help.site.com/hc/en/articles/1",
+        "https://help.site.com/pricing",
+        "https://www.site.com/all-integrations",
+        "https://site.com/blog/post",
+        "https://support.site.com/x",
+        "https://other.com/a",
+        "https://mailto:hi@help.site.com/hc/en/pay-by-bank",
+    ]
+    for mode in (SCOPE_HOST, SCOPE_PATH):
+        scope = CrawlScope.for_seed(seed, mode)
+        for url in urls:
+            assert pruner.in_scope(url, seed, mode) == scope.allows(url), (mode, url)
+
+
+def test_pruner_leaves_non_crawled_pages_alone():
+    """Text pages and PDF chunks live in the same table under their own ids —
+    they are not crawled URLs, so the pruner must never select them."""
+    pruner = _pruner()
+    seed = "https://help.site.com/hc/en"
+    assert pruner.in_scope("Shipping Policy", seed, SCOPE_HOST)
+    assert pruner.in_scope("7acf323c-7100-4e34-b0db-bcda9f5b2ef1_1", seed, SCOPE_PATH)
+    # ...while chunk ids of a crawled page still resolve to their page.
+    assert pruner.page_id_of("https://site.com/docs_2") == "https://site.com/docs"
+    assert pruner.page_id_of("https://site.com/getting_started") == "https://site.com/getting_started"
+
+
+def test_malformed_urls_are_out_of_scope_not_crashes():
+    scope = CrawlScope.for_seed("https://site.com/")
+    assert not scope.allows("https://[not-an-ipv6/x")
+    assert not scope.allows("")
+    # A seed we can't parse a host from allows nothing (host/path modes).
+    assert not CrawlScope.for_seed("https://[bad").allows("https://site.com/a")

backend/tests/knowledge/test_crawler_identity.py

@@ -0,0 +1,49 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The crawler says who it is, the same way everywhere it fetches - and a site
+that refuses it gets advice that says so, not a guess about bot protection.
+"""
+
+import inspect
+
+from app.knowledge import crawl4ai_fallback
+from app.knowledge.crawler_identity import CRAWLER_USER_AGENT
+from app.knowledge.enhanced_website_reader import EnhancedWebsiteReader
+from app.knowledge.failure_advice import FAILURE_ADVICE, FailureCause, plain_reason
+
+
+def test_identifies_honestly_by_name():
+    assert "ChatterMateBot" in CRAWLER_USER_AGENT
+    assert "Chrome/" not in CRAWLER_USER_AGENT  # the stale fake that WordPress.com refuses
+
+
+def test_reader_sends_the_shared_identity():
+    reader = EnhancedWebsiteReader()
+    assert reader.user_agent == CRAWLER_USER_AGENT
+    assert reader.headers["User-Agent"] == CRAWLER_USER_AGENT
+
+
+def test_browser_fallback_uses_the_shared_identity():
+    source = inspect.getsource(crawl4ai_fallback)
+    assert "Chrome/120" not in source
+    assert source.count("user_agent=CRAWLER_USER_AGENT") == 2  # both browser paths
+
+
+def test_a_refusal_reads_as_a_refusal():
+    url = "https://chattermate.chat"
+    reason = plain_reason(FAILURE_ADVICE[FailureCause.BOT_BLOCKED].format(url=url), url)
+    assert reason.startswith("This site refused our crawler.")
+    assert "challenge page" not in reason

backend/tests/knowledge/test_domains.py

@@ -0,0 +1,47 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from app.knowledge.domains import domain_of_url, registrable_domain
+
+
+def test_registrable_domain_basics():
+    assert registrable_domain("www.site.com") == "site.com"
+    assert registrable_domain("blog.site.com") == "site.com"
+    assert registrable_domain("SITE.com") == "site.com"
+    assert registrable_domain("") == ""
+    assert registrable_domain("localhost") == "localhost"
+
+
+def test_registrable_domain_cctld_aware():
+    # ccTLDs keep three labels so distinct *.co.uk are NOT collapsed to 'co.uk'.
+    assert registrable_domain("a.b.example.co.uk") == "example.co.uk"
+    assert registrable_domain("mycompany.co.uk") == "mycompany.co.uk"
+    assert registrable_domain("competitor.co.uk") == "competitor.co.uk"
+    assert registrable_domain("shop.example.com.au") == "example.com.au"
+
+
+def test_domain_of_url_normalizes_and_strips_port():
+    assert domain_of_url("https://www.site.com:8443/x") == "site.com"
+    # Scheme-less values still resolve their host (can't slip past as a path).
+    assert domain_of_url("evil.com/page") == "evil.com"
+    assert domain_of_url("site.com") == "site.com"
+    assert domain_of_url("") == ""
+
+
+def test_domain_of_url_distinguishes_cctld_domains():
+    # The add-subpage guard relies on these being DIFFERENT.
+    assert domain_of_url("https://mycompany.co.uk") != domain_of_url("https://competitor.co.uk")
+    assert domain_of_url("https://mycompany.co.uk/a") == domain_of_url("blog.mycompany.co.uk/b")

backend/tests/knowledge/test_enhanced_pdf_kb.py

@@ -0,0 +1,238 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import pytest
+from unittest.mock import MagicMock, patch, PropertyMock
+from agno.document import Document
+
+from app.knowledge.enhanced_pdf_kb import EnhancedPDFKnowledgeBase
+
+
+@pytest.fixture
+def mock_vector_db():
+    """Create a mock vector database"""
+    db = MagicMock()
+    db.exists.return_value = True
+    db.embedder = MagicMock()
+    return db
+
+
+@pytest.fixture
+def mock_reader():
+    """Create a mock PDF reader"""
+    reader = MagicMock()
+    return reader
+
+
+@pytest.fixture
+def mock_document():
+    """Create a mock document"""
+    doc = MagicMock(spec=Document)
+    doc.id = "test_doc_1"
+    doc.embedding = None
+    return doc
+
+
+class TestEnhancedPDFKnowledgeBase:
+    """Tests for EnhancedPDFKnowledgeBase class"""
+
+    def test_load_no_vector_db(self):
+        """Test load when no vector db is provided"""
+        kb = EnhancedPDFKnowledgeBase(path="/test/path")
+        kb.vector_db = None
+
+        # Should return early without error
+        kb.load()
+
+    def test_load_no_reader(self, mock_vector_db):
+        """Test load when no reader is provided"""
+        kb = EnhancedPDFKnowledgeBase(path="/test/path")
+        kb.vector_db = mock_vector_db
+        kb.reader = None
+
+        # Should return early without error
+        kb.load()
+
+    def test_load_recreate_db(self, mock_vector_db, mock_reader):
+        """Test load with recreate=True"""
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = []
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(recreate=True)
+
+            mock_vector_db.drop.assert_called_once()
+            mock_vector_db.create.assert_called_once()
+
+    def test_load_create_if_not_exists(self, mock_vector_db, mock_reader):
+        """Test load creates db if it doesn't exist"""
+        mock_vector_db.exists.return_value = False
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = []
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(recreate=False)
+
+            mock_vector_db.create.assert_called_once()
+
+    def test_load_with_documents(self, mock_vector_db, mock_reader, mock_document):
+        """Test load with documents"""
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[mock_document]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(upsert=True)
+
+            # Document should be embedded
+            mock_document.embed.assert_called_once_with(embedder=mock_vector_db.embedder)
+            # Documents should be upserted
+            mock_vector_db.upsert.assert_called_once()
+
+    def test_load_with_filters(self, mock_vector_db, mock_reader, mock_document):
+        """Test load with metadata filters"""
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[mock_document]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            filters = {"organization_id": "test_org", "agent_id": "test_agent"}
+            kb.load(upsert=True, filters=filters)
+
+            mock_vector_db.upsert.assert_called_once_with(
+                documents=[mock_document],
+                filters=filters
+            )
+
+    def test_load_document_already_embedded(self, mock_vector_db, mock_reader):
+        """Test load when document is already embedded"""
+        doc = MagicMock(spec=Document)
+        doc.id = "test_doc_1"
+        doc.embedding = [0.1, 0.2, 0.3]  # Already has embedding
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[doc]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(upsert=True)
+
+            # embed should not be called since doc already has embedding
+            doc.embed.assert_not_called()
+
+    def test_load_embedding_error(self, mock_vector_db, mock_reader, mock_document):
+        """Test load when embedding fails"""
+        mock_document.embed.side_effect = Exception("Embedding error")
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[mock_document]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            # Should not raise, but log error
+            kb.load(upsert=True)
+
+    def test_load_upsert_error(self, mock_vector_db, mock_reader, mock_document):
+        """Test load when upsert fails"""
+        mock_vector_db.upsert.side_effect = Exception("Upsert error")
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[mock_document]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            # Should not raise, but log error
+            kb.load(upsert=True)
+
+    def test_load_skip_upsert(self, mock_vector_db, mock_reader, mock_document):
+        """Test load with upsert=False"""
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[mock_document]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(upsert=False)
+
+            # upsert should not be called
+            mock_vector_db.upsert.assert_not_called()
+
+    def test_load_multiple_document_lists(self, mock_vector_db, mock_reader):
+        """Test load with multiple document lists"""
+        doc1 = MagicMock(spec=Document)
+        doc1.id = "doc1"
+        doc1.embedding = None
+
+        doc2 = MagicMock(spec=Document)
+        doc2.id = "doc2"
+        doc2.embedding = None
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[doc1], [doc2]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_vector_db
+            kb.reader = mock_reader
+
+            kb.load(upsert=True)
+
+            # Both documents should be embedded
+            doc1.embed.assert_called_once()
+            doc2.embed.assert_called_once()
+
+            # upsert should be called twice (once per document list)
+            assert mock_vector_db.upsert.call_count == 2
+
+    def test_load_no_embedder(self, mock_reader):
+        """Test load when vector_db has no embedder"""
+        mock_db = MagicMock()
+        mock_db.exists.return_value = True
+        # No embedder attribute
+        del mock_db.embedder
+
+        doc = MagicMock(spec=Document)
+        doc.id = "doc1"
+        doc.embedding = None
+
+        with patch.object(EnhancedPDFKnowledgeBase, 'document_lists', new_callable=PropertyMock) as mock_doc_lists:
+            mock_doc_lists.return_value = [[doc]]
+
+            kb = EnhancedPDFKnowledgeBase(path="/test/path")
+            kb.vector_db = mock_db
+            kb.reader = mock_reader
+
+            kb.load(upsert=True)
+
+            # embed should not be called since no embedder
+            doc.embed.assert_not_called()

backend/tests/knowledge/test_enhanced_website_kb.py

@@ -0,0 +1,500 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import pytest
+import os
+from unittest.mock import MagicMock, patch, call, ANY
+from agno.document.base import Document
+from app.core.config import settings
+from app.knowledge.crawl_scope import DEFAULT_CRAWL_SCOPE
+from app.knowledge.deadline import Deadline, SourceTimedOut
+from app.knowledge.enhanced_website_kb import EnhancedWebsiteKnowledgeBase
+from app.knowledge.failure_advice import FAILURE_ADVICE
+from app.knowledge.enhanced_website_reader import (
+    BotProtectionError,
+    EmptyCrawlError,
+    EnhancedWebsiteReader,
+    FailureCause,
+)
+
+# Test Data
+TEST_URLS = [
+    "https://example.com",
+    "https://test.com"
+]
+
+def _embedded(content, url):
+    """A document as it looks once a real run has embedded it.
+
+    Documents without an embedding are skipped by the vector store, so a fixture
+    without one describes a source that stores nothing — which the guard now
+    (correctly) fails.
+    """
+    document = Document(content=content, meta_data={"url": url})
+    document.embedding = [0.1, 0.2, 0.3]
+    return document
+
+
+TEST_DOCUMENTS = [
+    _embedded("Test content 1", "https://example.com/page1"),
+    _embedded("Test content 2", "https://example.com/page2"),
+]
+
+@pytest.fixture
+def mock_vector_db():
+    """Create a mock vector database"""
+    mock_db = MagicMock()
+    mock_db.upsert_available.return_value = True
+    mock_db.name_exists.return_value = False
+    mock_db.doc_exists.return_value = False
+    return mock_db
+
+@pytest.fixture
+def mock_reader():
+    """Create a mock website reader"""
+    mock = MagicMock(spec=EnhancedWebsiteReader)
+    mock.read.return_value = TEST_DOCUMENTS
+    # The real reader declares this as an int defaulting to 0; left as a Mock it
+    # is truthy, which would read as "bot protection blocked every page".
+    mock._challenge_blocked = 0
+    return mock
+
+class TestEnhancedWebsiteKnowledgeBase:
+    def test_initialization_with_defaults(self):
+        """Test initialization with default values"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS)
+
+        # Against settings, not literals: every one of these is configurable per
+        # install (KB_MAX_DEPTH and friends), so hardcoding the defaults here
+        # fails on any machine whose .env tunes a crawl.
+        assert kb.urls == TEST_URLS
+        assert isinstance(kb.reader, EnhancedWebsiteReader)
+        assert kb.max_depth == settings.KB_MAX_DEPTH
+        assert kb.max_links == settings.KB_MAX_LINKS
+        assert kb.min_content_length == settings.KB_MIN_CONTENT_LENGTH
+        assert kb.timeout == settings.KB_TIMEOUT
+        assert kb.max_retries == settings.KB_MAX_RETRIES
+        assert kb.max_workers == settings.KB_MAX_WORKERS
+        assert kb.crawl_scope == DEFAULT_CRAWL_SCOPE
+    
+    def test_initialization_with_custom_params(self):
+        """Test initialization with custom parameters"""
+        kb = EnhancedWebsiteKnowledgeBase(
+            urls=TEST_URLS,
+            max_depth=5,
+            max_links=20,
+            min_content_length=200,
+            timeout=60,
+            max_retries=5,
+            max_workers=8
+        )
+        
+        assert kb.urls == TEST_URLS
+        assert isinstance(kb.reader, EnhancedWebsiteReader)
+        assert kb.max_depth == 5
+        assert kb.max_links == 20
+        assert kb.min_content_length == 200
+        assert kb.timeout == 60
+        assert kb.max_retries == 5
+        assert kb.max_workers == 8
+        
+        # Verify reader was initialized with custom params
+        assert kb.reader.max_depth == 5
+        assert kb.reader.max_links == 20
+        assert kb.reader.min_content_length == 200
+        assert kb.reader.timeout == 60
+        assert kb.reader.max_retries == 5
+        assert kb.reader.max_workers == 8
+    
+    def test_initialization_with_custom_reader(self):
+        """Test initialization with a custom reader"""
+        custom_reader = EnhancedWebsiteReader(max_depth=4, max_links=15)
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=custom_reader)
+
+        assert kb.reader is custom_reader
+        assert kb.reader.max_depth == 4
+        assert kb.reader.max_links == 15
+
+    def test_raise_if_nothing_stored(self):
+        """A run that stored no pages must raise so the queue item fails.
+
+        Previously only the bot-blocked case raised; an empty crawl for any other
+        reason was marked COMPLETED, making a broken source look identical to a
+        healthy one with no sub-pages.
+        """
+        reader = EnhancedWebsiteReader(max_links=5)
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=reader)
+
+        # Anything stored → never raises, whatever happened along the way.
+        reader._challenge_blocked = 0
+        kb._raise_if_nothing_stored(5)
+        reader._challenge_blocked = 3
+        kb._raise_if_nothing_stored(2)
+
+        # Zero stored because bot protection blocked every page → the specific,
+        # actionable error, which takes precedence.
+        reader._challenge_blocked = 3
+        with pytest.raises(BotProtectionError):
+            kb._raise_if_nothing_stored(0)
+
+        # Zero stored for any other reason → still a failure, not a success.
+        reader._challenge_blocked = 0
+        with pytest.raises(EmptyCrawlError) as excinfo:
+            kb._raise_if_nothing_stored(0)
+        # The message names the source so the dashboard error is actionable.
+        assert TEST_URLS[0] in str(excinfo.value)
+
+    def test_a_crawled_page_that_never_reached_the_vector_store_is_a_failure(self):
+        """Crawled and stored are different numbers.
+
+        chatring.ai crawled one document, failed to embed it, stored nothing,
+        and was still marked COMPLETED — the org's vector table was empty while
+        the dashboard showed a healthy source. The guard counts what survived
+        the upsert, so a batch that stored nothing still fails the run.
+        """
+        reader = EnhancedWebsiteReader(max_links=5)
+        reader._challenge_blocked = 0
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=reader)
+
+        with pytest.raises(EmptyCrawlError):
+            kb._raise_if_nothing_stored(0)
+
+    def test_failed_batch_reports_zero_stored(self, mock_vector_db):
+        """An upsert that raises must not be counted as progress."""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS)
+        kb.vector_db = mock_vector_db
+
+        assert kb._process_document_batch(TEST_DOCUMENTS) == len(TEST_DOCUMENTS)
+
+        mock_vector_db.upsert.side_effect = RuntimeError("connection reset")
+        assert kb._process_document_batch(TEST_DOCUMENTS) == 0
+
+    def test_a_run_whose_documents_never_embedded_fails(self, mock_vector_db, mock_reader):
+        """Crawling is not storing.
+
+        chatring.ai crawled one document, failed to embed it, stored nothing, and
+        was still marked COMPLETED — the org's vector table was empty while the
+        dashboard showed a healthy source. A document with no embedding is
+        skipped by the vector store, so the run has produced nothing retrievable
+        and must fail.
+        """
+        unembedded = Document(content="crawled but never embedded", meta_data={})
+        unembedded.embedding = None
+        mock_reader.read.return_value = [unembedded]
+
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS[:1], reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        mock_vector_db.embedder = None
+
+        with pytest.raises(EmptyCrawlError):
+            kb.load()
+
+    @pytest.mark.parametrize("cause,expected", [
+        (FailureCause.DNS, "does not resolve"),
+        (FailureCause.TIMEOUT, "did not respond in time"),
+        (FailureCause.BOT_BLOCKED, "refused our crawler"),
+        (FailureCause.EMPTY_RESPONSE, "returned an empty page"),
+        (FailureCause.THIN_CONTENT, "JavaScript"),
+        (FailureCause.HTTP_ERROR, "error response"),
+        (FailureCause.BLOCKED_HOST, "not allowed to fetch"),
+    ])
+    def test_the_message_names_the_cause_the_crawl_actually_hit(self, cause, expected):
+        """Every distinct cause gets its own advice.
+
+        One string covering "empty, require JavaScript, or be unreachable" was
+        undiagnosable for the user and for us; the reader already knew which of
+        the three it hit.
+        """
+        reader = EnhancedWebsiteReader(max_links=5)
+        reader._challenge_blocked = 0
+        reader._failure_causes = {cause: 2}
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=reader)
+
+        with pytest.raises(EmptyCrawlError) as excinfo:
+            kb._raise_if_nothing_stored(0)
+
+        message = str(excinfo.value)
+        assert expected in message
+        assert TEST_URLS[0] in message
+
+    def test_message_falls_back_when_no_cause_was_recorded(self):
+        """An unexplained empty crawl still says something sensible."""
+        reader = EnhancedWebsiteReader(max_links=5)
+        reader._challenge_blocked = 0
+        reader._failure_causes = {}
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=reader)
+
+        with pytest.raises(EmptyCrawlError) as excinfo:
+            kb._raise_if_nothing_stored(0)
+        assert "No content could be read" in str(excinfo.value)
+
+    def test_nothing_stored_because_everything_was_already_indexed(self):
+        """Skipping URLs we already hold is a no-op, not an unreadable source.
+
+        load() drops URLs already in the vector store, so a re-add reaches this
+        guard having stored nothing. Raising there failed the whole source and
+        blamed the site — "may require JavaScript" — for content already indexed.
+        """
+        reader = EnhancedWebsiteReader(max_links=5)
+        reader._challenge_blocked = 0
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=reader)
+
+        # Every URL skipped as already present → nothing to do, no error.
+        kb._raise_if_nothing_stored(0, skipped_existing=len(TEST_URLS))
+
+        # Some URLs were still meant to be read and yielded nothing → real failure.
+        with pytest.raises(EmptyCrawlError):
+            kb._raise_if_nothing_stored(0, skipped_existing=len(TEST_URLS) - 1)
+
+        # Bot protection keeps precedence over the skip shortcut: its message is
+        # the actionable one, and a blocked sitemap must still fail loudly.
+        reader._challenge_blocked = 2
+        with pytest.raises(BotProtectionError):
+            kb._raise_if_nothing_stored(0, skipped_existing=len(TEST_URLS))
+    
+    def test_document_lists_property(self, mock_reader):
+        """Test the document_lists property"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        
+        # Convert iterator to list to verify contents
+        doc_lists = list(kb.document_lists)
+        
+        assert len(doc_lists) == len(TEST_URLS)
+        assert all(docs == TEST_DOCUMENTS for docs in doc_lists)
+        assert mock_reader.read.call_count == len(TEST_URLS)
+        
+        # Verify read was called with correct URLs and new callback args
+        for url in TEST_URLS:
+            mock_reader.read.assert_any_call(
+                url=url,
+                vector_db_callback=ANY,
+                url_crawled_callback=ANY
+            )
+    
+    def test_load_without_vector_db(self, mock_reader):
+        """Test load method when no vector db is provided"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        
+        with patch('app.knowledge.enhanced_website_kb.logger') as mock_logger:
+            kb.load()
+            mock_logger.warning.assert_called_with("No vector db provided")
+    
+    def test_load_with_recreate(self, mock_vector_db, mock_reader):
+        """Test load method with recreate=True"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        
+        # Set up mock reader to return different documents for each URL
+        doc1 = _embedded("Test content 1", "https://example.com/page1")
+        doc2 = _embedded("Test content 2", "https://test.com/page1")
+        mock_reader.read.side_effect = [[doc1], [doc2]]
+        
+        kb.load(recreate=True)
+        
+        # Verify vector db operations
+        mock_vector_db.drop.assert_called_once()
+        mock_vector_db.create.assert_called_once()
+        
+        # Verify reader operations
+        assert mock_reader.read.call_count == len(TEST_URLS)
+        mock_reader.read.assert_has_calls([
+            call(url=TEST_URLS[0], vector_db_callback=ANY, url_crawled_callback=ANY),
+            call(url=TEST_URLS[1], vector_db_callback=ANY, url_crawled_callback=ANY)
+        ], any_order=True)
+        
+        # Verify batch upsert was called with combined documents
+        mock_vector_db.upsert.assert_called_once()
+        # Get the actual documents passed to upsert
+        actual_docs = mock_vector_db.upsert.call_args[1]['documents']
+        assert len(actual_docs) == 2
+        assert doc1 in actual_docs
+        assert doc2 in actual_docs
+
+    def test_load_with_existing_urls(self, mock_vector_db, mock_reader):
+        """Test load method with existing URLs in vector db"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        
+        # Simulate first URL already exists
+        mock_vector_db.name_exists.side_effect = lambda name: name == TEST_URLS[0]
+        
+        kb.load(recreate=False)
+        
+        # Should only process second URL
+        assert mock_reader.read.call_count == 1
+        mock_reader.read.assert_called_once_with(
+            url=TEST_URLS[1],
+            vector_db_callback=ANY,
+            url_crawled_callback=ANY
+        )
+        
+        # Verify upsert was called once with the documents
+        mock_vector_db.upsert.assert_called_once()
+
+    def test_load_when_every_url_is_already_indexed(self, mock_vector_db, mock_reader):
+        """Re-adding a source that is fully indexed must not fail it.
+
+        Nothing is crawled and nothing is stored, which used to surface on the
+        queue item as "No content could be read ... may require JavaScript" even
+        though the content was sitting in the vector store the whole time.
+        """
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        mock_vector_db.name_exists.return_value = True
+
+        kb.load(recreate=False)
+
+        mock_reader.read.assert_not_called()
+
+    def test_load_with_existing_documents(self, mock_vector_db, mock_reader):
+        """Test load method with existing documents"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        
+        # Simulate first document already exists
+        mock_vector_db.doc_exists.side_effect = lambda doc: doc == TEST_DOCUMENTS[0]
+        
+        kb.load(recreate=False)
+        
+        # Verify that batch upsert is used with all documents
+        mock_vector_db.upsert.assert_called_once()
+        # Extract the documents passed to upsert
+        actual_docs = mock_vector_db.upsert.call_args[1]['documents']
+        assert len(actual_docs) >= 1  # We should have at least one document
+    
+    def test_load_without_upsert(self, mock_vector_db, mock_reader):
+        """Test load method when upsert is not available"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        
+        # Simulate upsert not available
+        mock_vector_db.upsert_available.return_value = False
+        
+        kb.load(upsert=False)
+        
+        # Verify reader was called
+        assert mock_reader.read.call_count > 0
+        
+        # When upsert is False, no upsert should be called
+        mock_vector_db.upsert.assert_not_called()
+    
+    def test_load_never_builds_an_approximate_index(self, mock_vector_db, mock_reader):
+        """Agent searches filter by agent_id; an HNSW/IVFFlat index would filter
+        after choosing candidates and could return fewer results. Exact search only."""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+
+        kb.load()
+
+        mock_vector_db.optimize.assert_not_called()
+    
+    def test_load_with_error_handling(self, mock_vector_db, mock_reader):
+        """Test load method error handling"""
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS, reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        
+        # Simulate error for first URL
+        mock_reader.read.side_effect = [Exception("Test error"), TEST_DOCUMENTS]
+        
+        # Disable embedding for this test to avoid extra error logging
+        mock_vector_db.embedder = None
+        
+        with patch('app.knowledge.enhanced_website_kb.logger') as mock_logger:
+            kb.load()
+            
+            # Should log error and continue with second URL
+            # Only check for the specific error we're expecting
+            mock_logger.error.assert_any_call("Error reading URL https://example.com: Test error")
+            
+            # Should have still processed the second URL
+            assert mock_reader.read.call_count == 2
+            
+            # Should still have called upsert with the successful documents
+            mock_vector_db.upsert.assert_called_once() 
+
+class ExpiringDeadline:
+    expired = False
+
+
+class CountingEmbedder:
+    def __init__(self):
+        self.calls = 0
+
+    def get_embedding_and_usage(self, text):
+        self.calls += 1
+        return [0.1, 0.2, 0.3], None
+
+
+class TestASourceThatRunsOutOfTime:
+    """Seen in production: one source ran for three hours until the worker hit
+    its memory limit. Past its time limit a source fails, and stores nothing:
+    a partial copy would make adding it again skip the crawl."""
+
+    def test_it_fails_with_advice_and_stores_nothing(self, mock_vector_db, mock_reader):
+        deadline = ExpiringDeadline()
+
+        def slow_read(**kwargs):
+            deadline.expired = True
+            return TEST_DOCUMENTS
+        mock_reader.read.side_effect = slow_read
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS[:1], reader=mock_reader)
+        kb.vector_db = mock_vector_db
+
+        with patch('app.knowledge.enhanced_website_kb.Deadline', return_value=deadline), \
+             pytest.raises(SourceTimedOut) as raised:
+            kb.load()
+
+        assert str(raised.value) == FAILURE_ADVICE[FailureCause.TOO_LONG].format(url=TEST_URLS[0])
+        mock_vector_db.upsert.assert_not_called()
+        assert mock_reader.deadline is deadline  # the crawl stops at the same time
+
+    def test_its_queued_chunks_are_not_embedded(self, mock_vector_db, mock_reader):
+        deadline = ExpiringDeadline()
+        embedder = CountingEmbedder()
+        mock_vector_db.embedder = embedder
+        chunks = [Document(content=f"chunk {i}", meta_data={"url": "https://example.com"}) for i in range(200)]
+
+        def slow_read(url, vector_db_callback=None, url_crawled_callback=None):
+            deadline.expired = True
+            for chunk in chunks:
+                vector_db_callback(chunk)
+            return chunks
+        mock_reader.read.side_effect = slow_read
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS[:1], reader=mock_reader)
+        kb.vector_db = mock_vector_db
+
+        with patch('app.knowledge.enhanced_website_kb.Deadline', return_value=deadline), \
+             pytest.raises(SourceTimedOut):
+            kb.load()
+
+        assert embedder.calls == 0
+
+    def test_a_source_read_in_time_is_stored(self, mock_vector_db, mock_reader):
+        mock_vector_db.embedder = CountingEmbedder()
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS[:1], reader=mock_reader)
+        kb.vector_db = mock_vector_db
+
+        kb.load()
+
+        mock_vector_db.upsert.assert_called_once()
+        assert isinstance(mock_reader.deadline, Deadline)
+
+    def test_the_limit_comes_from_settings(self, mock_vector_db, mock_reader):
+        kb = EnhancedWebsiteKnowledgeBase(urls=TEST_URLS[:1], reader=mock_reader)
+        kb.vector_db = mock_vector_db
+        with patch('app.knowledge.enhanced_website_kb.Deadline', wraps=Deadline) as made:
+            kb.load()
+        made.assert_called_once_with(settings.KB_SOURCE_TIME_LIMIT_SECONDS)

backend/tests/knowledge/test_enhanced_website_reader.py

@@ -0,0 +1,780 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import socket
+import unittest
+from unittest.mock import patch, MagicMock, Mock
+from bs4 import BeautifulSoup
+import httpx
+
+from app.knowledge.crawl_scope import DEFAULT_CRAWL_SCOPE, CrawlScope
+from app.knowledge.enhanced_website_reader import (
+    EnhancedWebsiteReader,
+    FAILURE_ADVICE,
+    FailureCause,
+)
+
+
+_RealClient = httpx.Client
+
+
+def serving(handler):
+    """The reader's HTTP client, answering every request with `handler(request)`:
+    real httpx responses, headers and streaming included."""
+    return patch(
+        'app.knowledge.enhanced_website_reader.httpx.Client',
+        side_effect=lambda **kwargs: _RealClient(transport=httpx.MockTransport(handler), **kwargs),
+    )
+
+
+class TestEnhancedWebsiteReader(unittest.TestCase):
+    """Test cases for EnhancedWebsiteReader"""
+
+    def setUp(self):
+        """Set up test environment"""
+        self.reader = EnhancedWebsiteReader(
+            max_depth=2,
+            max_links=5,
+            min_content_length=50
+        )
+
+        # The fetch path now runs an SSRF guard (url_safety.resolves_to_blocked_host)
+        # which does a DNS lookup — stub it to a fixed public IP so tests don't hit
+        # the network and aren't blocked.
+        dns_patcher = patch(
+            'app.knowledge.url_safety.socket.getaddrinfo',
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
+        )
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+        
+        # Create a simple HTML response for testing
+        self.test_html = """
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <title>Test Page</title>
+            <style>
+                .hidden { display: none; }
+            </style>
+            <script>console.log('This should be ignored');</script>
+        </head>
+        <body>
+            <header>
+                <nav>
+                    <a href="/page1">Page 1</a>
+                    <a href="/page2">Page 2</a>
+                </nav>
+            </header>
+            <div class="banner">Banner content to be ignored</div>
+            <main>
+                <h1>Main Content</h1>
+                <p>This is the main content of the page. It should be extracted properly.</p>
+                <p>Additional paragraph with meaningful content.</p>
+            </main>
+            <div class="sidebar">
+                <h2>Sidebar</h2>
+                <p>Sidebar content should be ignored.</p>
+            </div>
+            <div class="content">
+                <h2>Additional Content</h2>
+                <p>More content in a div with class 'content'.</p>
+            </div>
+            <div id="post-content">
+                <h2>Post Content</h2>
+                <p>Content in a div with id 'post-content'.</p>
+            </div>
+            <div>
+                <h2>Generic Content</h2>
+                <p>Content in a generic div without special class or id.</p>
+                <p>This is a good paragraph with substantial text that should be detected by density extraction.</p>
+                <p>Another good paragraph that helps identify this div as having high text density.</p>
+            </div>
+            <footer>
+                <p>Footer content to be ignored.</p>
+            </footer>
+            <div class="hidden">
+                <p>This should be ignored because it's hidden.</p>
+            </div>
+        </body>
+        </html>
+        """
+        self.soup = BeautifulSoup(self.test_html, 'html.parser')
+        
+    def test_extracts_every_content_block_on_the_page(self):
+        """All of a page's content blocks are kept, not just the first one.
+
+        The old walk returned the first container clearing min_content_length,
+        so a page like this one yielded only <main> and silently dropped the
+        three sibling blocks below it.
+        """
+        content = self.reader._extract_main_content(self.soup)
+        self.assertIn("This is the main content of the page", content)
+        self.assertIn("More content in a div with class 'content'", content)
+        self.assertIn("Content in a div with id 'post-content'", content)
+        self.assertIn("Content in a generic div", content)
+
+    def test_a_banner_before_the_content_does_not_win(self):
+        """A short block that merely clears the floor must not beat the page.
+
+        This is eazzyliving.co.uk: its first <article> was a 108-character
+        search widget, and returning it discarded ~4,500 characters of page.
+        The customer retried four times and left.
+        """
+        html = """
+        <html><body>
+            <article>Search properties. Search by city, university, or
+            neighbourhood to find verified student rooms and studios.</article>
+            <div class="listings">
+                <h1>Student accommodation in Leeds</h1>
+                <p>%s</p>
+            </div>
+        </body></html>
+        """ % ("Verified rooms close to campus with bills included. " * 40)
+
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Student accommodation in Leeds", content)
+        self.assertIn("Verified rooms close to campus", content)
+        self.assertGreater(len(content), 1000)
+
+    def test_the_page_title_survives_when_it_sits_in_a_page_header(self):
+        """Docs themes put the <h1> in a <header>, which selection treats as chrome.
+
+        docs.chattermate.chat lost "Quickstart: Launch Your AI Support Agent in
+        5 Minutes" that way. A page's title is its strongest retrieval signal,
+        so losing it makes the page harder for an agent to find, not just
+        shorter.
+        """
+        body = "Run the stack on your own infrastructure with the CLI. " * 20
+        html = f"""
+        <html><body>
+            <header><h1>Quickstart: Launch in 5 Minutes</h1></header>
+            <div class="prose"><p>{body}</p></div>
+            <footer>GitHub Discord</footer>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Quickstart: Launch in 5 Minutes", content)
+        self.assertIn("own infrastructure", content)
+        self.assertNotIn("Discord", content, "footer chrome should stay out")
+
+    def test_a_page_with_no_h1_falls_back_to_the_document_title(self):
+        """<title> lives in <head>, which _clean_soup strips.
+
+        Reading it after the clean would always find nothing, so a page with no
+        <h1> would silently have no title at all.
+        """
+        body = "Refund terms and conditions apply to all orders. " * 20
+        html = f"""
+        <html><head><title>Refund Policy</title></head>
+        <body><div class="c"><p>{body}</p></div></body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertIn("Refund Policy", content)
+
+    def test_the_title_is_not_duplicated_when_already_in_the_content(self):
+        """The usual case: the <h1> is inside the content container already."""
+        body = "Every plan we offer, billed monthly or yearly. " * 20
+        html = f"""
+        <html><body>
+            <main><h1>Plans and pricing</h1><p>{body}</p></main>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Plans and pricing", content)
+        self.assertEqual(content.count("Plans and pricing"), 1,
+                         "the heading must not be prepended on top of itself")
+
+    def test_a_page_of_only_boilerplate_stays_below_the_floor(self):
+        """Nav chrome must not pass as content.
+
+        www.solcontrol.ca renders its catalogue with JavaScript; all the served
+        HTML holds is a phone number and a login link. That cleared the
+        100-character floor, so it was stored as the page's content *and* it
+        suppressed the browser fallback that reads the site properly. Staying
+        under the floor is what routes the page to Crawl4AI in _process_url.
+        """
+        html = """
+        <html><body>
+            <header><a href="/login">Welcome, Guest - Login</a>
+                    <a href="tel:905-230-8468">905-230-8468</a></header>
+            <nav><a href="/a">Products</a><a href="/b">About</a></nav>
+            <footer><a href="mailto:info@solcontrol.ca">info@solcontrol.ca</a></footer>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertLess(len(content), self.reader.min_content_length)
+
+    def test_extract_content_by_density(self):
+        """Density extraction still runs when no container holds the page."""
+        html = """
+        <html><body>
+            <span>x</span>
+            <div><p>%s</p><p>%s</p></div>
+        </body></html>
+        """ % ("A good paragraph with substantial text. " * 5,
+               "Another good paragraph that carries the page. " * 5)
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertIn("A good paragraph with substantial text", content)
+
+    def test_clean_soup(self):
+        """Test cleaning of unwanted elements from HTML"""
+        # Create a copy for testing
+        soup_copy = BeautifulSoup(str(self.soup), 'html.parser')
+        self.reader._clean_soup(soup_copy)
+        
+        # Check that truly unwanted elements are removed (scripts, styles, hidden elements)
+        self.assertIsNone(soup_copy.find('script'))
+        self.assertIsNone(soup_copy.find('style'))
+        self.assertIsNone(soup_copy.find(class_='hidden'))
+        
+        # Check that navigation elements are removed (they contain menu links, not main content)
+        self.assertIsNone(soup_copy.find('nav'))
+        
+        # Check that sidebar elements are removed
+        self.assertIsNone(soup_copy.find(class_='sidebar'))
+        
+        # Check that main content elements are kept (header and footer may contain some content)
+        self.assertIsNotNone(soup_copy.find('header'))
+        self.assertIsNotNone(soup_copy.find('footer'))
+        self.assertIsNotNone(soup_copy.find('main'))
+        
+    def test_canonical_url(self):
+        """Fragments/trailing slashes are stripped so page variants collapse."""
+        c = self.reader._canonical_url
+        self.assertEqual(c('https://x.com/'), 'https://x.com')
+        self.assertEqual(c('https://x.com/#features'), 'https://x.com')
+        self.assertEqual(c('https://x.com/#pricing'), 'https://x.com')
+        self.assertEqual(c('https://x.com/pricing/'), 'https://x.com/pricing')
+        self.assertEqual(c('https://x.com/blogs#top'), 'https://x.com/blogs')
+        # The homepage and its anchored variants all canonicalize to one id.
+        variants = {c(u) for u in [
+            'https://x.com', 'https://x.com/', 'https://x.com/#a', 'https://x.com/#b'
+        ]}
+        self.assertEqual(len(variants), 1)
+
+    def test_looks_like_bot_challenge(self):
+        """Bot-check interstitials are detected so they aren't stored as content."""
+        f = self.reader._looks_like_bot_challenge
+        # The exact wp.com interstitial from the reported bug (strong marker).
+        self.assertTrue(f(
+            "Checking your browser This will only take a few seconds... "
+            "Secured by wp.com (URL: https://wordpress.com)"
+        ))
+        # Cloudflare markers.
+        self.assertTrue(f("cf-browser-verification"))
+        self.assertTrue(f("Just a moment..."))  # weak, but short
+        # Real content is not flagged, even if long and mentioning a weak phrase.
+        long_text = ("Our pricing is simple. " * 40) + "checking your browser settings is optional."
+        self.assertFalse(f(long_text))
+        self.assertFalse(f("Welcome to our pricing page. Plans start at $10 per seat."))
+        self.assertFalse(f(""))
+        self.assertFalse(f(None))
+
+    def test_extract_links_dedupes_page_variants(self):
+        """The homepage linked via #anchors and trailing slash yields one link."""
+        html = """
+        <html><body>
+          <a href="https://site.com/#features">Features</a>
+          <a href="https://site.com/#pricing">Pricing</a>
+          <a href="https://site.com/">Home</a>
+          <a href="https://site.com/docs">Docs</a>
+          <a href="https://site.com/docs/">Docs slash</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com')
+        # All homepage variants collapse (and equal the base, so are dropped);
+        # /docs and /docs/ collapse to a single canonical link.
+        self.assertEqual(links, ['https://site.com/docs'])
+
+    def test_extract_links_stays_on_the_seed_host_by_default(self):
+        """Default scope is the seed's own host: a help-center crawl must not
+        wander onto the marketing site, and a lookalike domain never matches."""
+        html = """
+        <html><body>
+          <a href="https://site.com/a">same</a>
+          <a href="https://www.site.com/b">www of the same host</a>
+          <a href="https://blog.site.com/c">subdomain (same registrable)</a>
+          <a href="https://evilsite.com/d">lookalike suffix</a>
+          <a href="https://other.com/e">unrelated</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com')
+        self.assertIn('https://site.com/a', links)
+        # In scope, and stored under the seed's spelling of the host.
+        self.assertIn('https://site.com/b', links)
+        self.assertNotIn('https://www.site.com/b', links)
+        self.assertNotIn('https://blog.site.com/c', links)
+        self.assertNotIn('https://evilsite.com/d', links)
+        self.assertNotIn('https://other.com/e', links)
+
+    def test_extract_links_domain_scope_spans_subdomains(self):
+        """The opt-in 'domain' scope keeps the previous reach across subdomains,
+        still by registrable-domain equality (never a suffix match)."""
+        reader = EnhancedWebsiteReader(crawl_scope='domain')
+        html = """
+        <html><body>
+          <a href="https://blog.site.com/b">subdomain</a>
+          <a href="https://evilsite.com/c">lookalike suffix</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = reader._extract_links(soup, 'https://site.com')
+        self.assertIn('https://blog.site.com/b', links)
+        self.assertNotIn('https://evilsite.com/c', links)
+
+    def test_extract_links_collapses_www_variants_onto_the_seed_host(self):
+        """A site that links both spellings stored the page twice — 4 of the 50
+        pages in a real paywithatoa.co.uk crawl were www/bare duplicates."""
+        html = """
+        <html><body>
+          <a href="https://www.site.com/terms">terms via www</a>
+          <a href="https://site.com/terms">terms bare</a>
+          <a href="https://www.site.com/privacy">privacy via www</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com/')
+        self.assertEqual(links, ['https://site.com/terms', 'https://site.com/privacy'])
+
+        # Seeded at the www spelling, the crawl keeps that one instead.
+        reader = EnhancedWebsiteReader()
+        links = reader._extract_links(soup, 'https://www.site.com/')
+        self.assertEqual(links, ['https://www.site.com/terms', 'https://www.site.com/privacy'])
+
+    def test_extract_links_skips_non_http_schemes(self):
+        """mailto:/tel: hrefs are contact actions, not pages. urljoin leaves them
+        untouched, and 'https://' + 'mailto:a@site.com' parses as host site.com
+        with 'mailto:a' as userinfo — so they used to be crawled as pages."""
+        html = """
+        <html><body>
+          <a href="mailto:hello@site.com">Email us</a>
+          <a href="tel:+441234567890">Call us</a>
+          <a href="javascript:void(0)">Menu</a>
+          <a href="/pricing">Pricing</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com/help')
+        self.assertEqual(links, ['https://site.com/pricing'])
+
+    def test_normalize_url_leaves_non_http_schemes_alone(self):
+        """Prefixing 'mailto:a@site.com' with https:// produced a fetchable
+        'https://mailto:a@site.com' — the source of the mangled stored pages."""
+        self.assertEqual(self.reader._normalize_url('mailto:a@site.com'), 'mailto:a@site.com')
+        self.assertEqual(self.reader._normalize_url('site.com/x'), 'https://site.com/x')
+        self.assertEqual(self.reader._normalize_url('http://site.com'), 'http://site.com')
+
+    def test_get_primary_domain_strips_port_and_www(self):
+        self.assertEqual(self.reader._get_primary_domain('https://www.site.com:8443/x'), 'site.com')
+        self.assertEqual(self.reader._get_primary_domain('https://a.b.example.co.uk/x'), 'example.co.uk')
+
+    def test_crawl_with_successful_request(self):
+        """Test crawling with successful HTTP requests"""
+        requests = []
+
+        def site(request):
+            requests.append(request)
+            return httpx.Response(200, html=self.test_html)
+
+        with serving(site):
+            result = self.reader.crawl('https://example.com')
+
+        self.assertTrue(requests)
+        self.assertIn('https://example.com', result)
+        self.assertIn("Main Content", result['https://example.com'])
+
+    @patch('time.sleep', return_value=None)  # Skip actual sleeping
+    def test_crawl_with_retries(self, mock_sleep):
+        """Test crawling with retries on failed requests"""
+        attempts = []
+
+        def flaky_home(request):
+            if request.url.path != "/":
+                return httpx.Response(404)
+            attempts.append(request)
+            if len(attempts) == 1:
+                return httpx.Response(500)  # HTTP error
+            if len(attempts) == 2:
+                raise httpx.ReadTimeout("slow", request=request)  # transport error
+            return httpx.Response(200, html=self.test_html)
+
+        with serving(flaky_home):
+            result = self.reader.crawl('https://example.com')
+
+        self.assertEqual(len(attempts), 3)
+        self.assertIn('https://example.com', result)
+        self.assertIn("Main Content", result['https://example.com'])
+
+    def test_read_method(self):
+        """Test the read method to ensure it returns proper Document objects"""
+        with serving(lambda request: httpx.Response(200, html=self.test_html)):
+            documents = self.reader.read('https://example.com')
+        
+        # Verify documents are created correctly
+        self.assertTrue(len(documents) > 0)
+        self.assertIn("Main Content", documents[0].content)
+        
+        # Verify metadata format
+        self.assertEqual(documents[0].meta_data['url'], 'https://example.com')
+        self.assertEqual(documents[0].meta_data['chunk'], 1)
+        self.assertIsInstance(documents[0].meta_data['chunk_size'], int)
+        
+        # Verify ID and name
+        self.assertEqual(documents[0].id, 'https://example.com')
+        # Verify that name is the original source URL
+        self.assertEqual(documents[0].name, 'https://example.com')
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+class TestUnreachableHostsFailFast(unittest.TestCase):
+    """A host we never reached fails the same way every attempt.
+
+    The old ladder — three tries with exponential backoff, then a headless
+    browser — spent 130 seconds on mateusoliveira.com (which drops the SYN on
+    80 and 443) and ~10 seconds on each domain with no A record, holding one of
+    only two queue slots for the whole time.
+    """
+
+    def setUp(self):
+        self.reader = EnhancedWebsiteReader(max_depth=1, max_links=1, min_content_length=50)
+        dns_patcher = patch(
+            'app.knowledge.url_safety.socket.getaddrinfo',
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
+        )
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def _crawl_with(self, error):
+        """Run one page through the fetch path with `error` raised every time."""
+        scope = CrawlScope.for_seed("https://example.com", DEFAULT_CRAWL_SCOPE)
+        with patch('app.knowledge.enhanced_website_reader.httpx.Client') as client, \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback') as browser, \
+             patch('app.knowledge.enhanced_website_reader.time.sleep') as sleep:
+            client.return_value.__enter__.return_value.stream.side_effect = error
+            browser.return_value.is_available = True
+            browser.return_value.fetch_with_browser.return_value = (None, None, None)
+            result = self.reader._process_url(("https://example.com", 1), scope)
+        return result, client.return_value.__enter__.return_value.stream, browser, sleep
+
+
+    def test_a_domain_that_does_not_resolve_is_tried_once(self):
+        error = httpx.ConnectError("[Errno 8] nodename nor servname provided, or not known")
+        result, get, browser, _ = self._crawl_with(error)
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1, "no point retrying a name that does not resolve")
+        browser.return_value.fetch_with_browser.assert_not_called()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.DNS: 1})
+
+    def test_a_dropped_connection_attempt_is_tried_once(self):
+        result, get, browser, _ = self._crawl_with(httpx.ConnectTimeout("timed out"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1)
+        browser.return_value.fetch_with_browser.assert_not_called()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.TIMEOUT: 1})
+
+    def test_a_refused_connection_still_gets_the_browser(self):
+        """Could be a TLS quirk rather than a dead host, so give Chromium a turn."""
+        result, get, browser, _ = self._crawl_with(httpx.ConnectError("Connection refused"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1, "a refusal will not change in two seconds")
+        browser.return_value.fetch_with_browser.assert_called_once()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.CONNECTION: 1})
+
+    def test_a_slow_page_is_still_retried(self):
+        """Read timeouts are the transient case retries exist for."""
+        result, get, browser, _ = self._crawl_with(httpx.ReadTimeout("too slow"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, self.reader.max_retries)
+        browser.return_value.fetch_with_browser.assert_called_once()
+
+    def test_our_own_bugs_are_not_blamed_on_the_customers_site(self):
+        """A crash in our parsing must not be reported as the site being down.
+
+        Everything inside the fetch block — BeautifulSoup, link extraction,
+        content selection — can raise. Classifying those as a connection
+        failure told the customer "your site refused the connection" when what
+        actually happened was our own AttributeError.
+        """
+        reader = EnhancedWebsiteReader()
+        for error in (AttributeError("'NoneType' has no attribute 'find'"),
+                      KeyError("href"),
+                      ValueError("bad parse")):
+            cause, _, _ = reader._classify_transport_error(error)
+            self.assertEqual(cause, FailureCause.UNKNOWN)
+            self.assertNotIn(cause, FAILURE_ADVICE,
+                             "UNKNOWN must fall back to the generic message")
+
+    def test_our_own_pool_exhaustion_is_not_blamed_on_the_site(self):
+        """PoolTimeout means we never sent the request, so the site is not slow."""
+        reader = EnhancedWebsiteReader()
+        cause, retry, _ = reader._classify_transport_error(httpx.PoolTimeout("pool full"))
+        self.assertEqual(cause, FailureCause.UNKNOWN)
+        self.assertTrue(retry, "a saturated pool does free up")
+
+    def test_name_resolution_is_detected_from_the_socket_error(self):
+        """Not from the message text, which other errors can imitate."""
+        reader = EnhancedWebsiteReader()
+
+        gai = httpx.ConnectError("connection failed")
+        gai.__cause__ = socket.gaierror(8, "nodename nor servname provided")
+        self.assertEqual(reader._classify_transport_error(gai)[0], FailureCause.DNS)
+
+        # A refusal that merely mentions a name must not be read as DNS, or we
+        # would skip the browser fallback for a host that is actually up.
+        refused = httpx.ConnectError("Connection refused by name-based vhost")
+        cause, _, browser = reader._classify_transport_error(refused)
+        self.assertEqual(cause, FailureCause.CONNECTION)
+        self.assertTrue(browser)
+
+
+class TestWwwApexFallback(unittest.TestCase):
+    """A ``www`` host with no address falls back to the apex.
+
+    Production job 392: a customer pasted https://www.utsabzone.store, whose
+    ``www`` CNAME is broken while the apex serves the site fine. They were told
+    "the domain does not resolve" — accurate about the name they gave us, and
+    useless, because the site was up.
+    """
+
+    def setUp(self):
+        self.reader = EnhancedWebsiteReader(max_depth=1, max_links=1)
+
+    def _resolver(self, resolvable):
+        """getaddrinfo that only knows about `resolvable` hosts."""
+        def fake(host, *args, **kwargs):
+            if host in resolvable:
+                return [(2, 1, 6, "", ("93.184.216.34", 0))]
+            raise socket.gaierror(8, "nodename nor servname provided, or not known")
+        return fake
+
+    def test_a_dead_www_seed_falls_back_to_the_apex(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"utsabzone.store"})):
+            out = self.reader._seed_with_resolvable_host("https://www.utsabzone.store")
+        self.assertEqual(out, "https://utsabzone.store")
+
+    def test_a_working_www_seed_is_left_alone(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"www.example.com", "example.com"})):
+            out = self.reader._seed_with_resolvable_host("https://www.example.com/pricing")
+        self.assertEqual(out, "https://www.example.com/pricing")
+
+    def test_a_non_www_seed_costs_no_lookup(self):
+        """The common case must not pay for DNS it does not need."""
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo') as gai:
+            out = self.reader._seed_with_resolvable_host("https://example.com")
+        self.assertEqual(out, "https://example.com")
+        gai.assert_not_called()
+
+    def test_both_hosts_dead_keeps_the_url_the_user_gave(self):
+        """So the error still names what they typed."""
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver(set())):
+            out = self.reader._seed_with_resolvable_host("https://www.jourico.com")
+        self.assertEqual(out, "https://www.jourico.com")
+
+    def test_the_path_query_and_port_survive_the_swap(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"shop.example"})):
+            out = self.reader._seed_with_resolvable_host(
+                "https://www.shop.example:8443/catalogue?page=2")
+        self.assertEqual(out, "https://shop.example:8443/catalogue?page=2")
+
+    def test_credentials_in_the_url_are_never_rewritten(self):
+        url = "https://user:pass@www.example.com/"
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"example.com"})):
+            self.assertEqual(self.reader._seed_with_resolvable_host(url), url)
+
+    def test_an_unverifiable_apex_is_never_swapped_in(self):
+        """A lookup that errors oddly must not count as "the apex is fine".
+
+        Otherwise we would rewrite the seed to a host we never confirmed and
+        report the failure against the wrong name.
+        """
+        def odd(host, *args, **kwargs):
+            if host.startswith("www."):
+                raise socket.gaierror(8, "not known")
+            raise UnicodeError("malformed host")
+
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo', side_effect=odd):
+            out = self.reader._seed_with_resolvable_host("https://www.example.com")
+        self.assertEqual(out, "https://www.example.com")
+
+    def test_the_swapped_apex_is_still_checked_by_the_ssrf_guard(self):
+        """The fallback derives a host the user never typed, so prove it is guarded.
+
+        A dead ``www`` whose apex points at loopback is the vector: without the
+        per-hop check in safe_get, the rewrite would reach an internal address.
+        """
+        from app.knowledge.url_safety import BlockedHostError, guard_url
+
+        def internal_apex(host, *args, **kwargs):
+            if host == "attacker.test":
+                return [(2, 1, 6, "", ("127.0.0.1", 0))]
+            raise socket.gaierror(8, "nodename nor servname provided")
+
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=internal_apex):
+            swapped = self.reader._seed_with_resolvable_host("https://www.attacker.test")
+        self.assertEqual(swapped, "https://attacker.test")
+
+        with patch('app.knowledge.url_safety.socket.getaddrinfo', side_effect=internal_apex):
+            with self.assertRaises(BlockedHostError):
+                guard_url(swapped)
+
+    def test_the_swapped_apex_stays_inside_the_crawl_scope(self):
+        """Otherwise the fallback would fetch one page and refuse every link."""
+        scope = CrawlScope.for_seed("https://www.utsabzone.store", DEFAULT_CRAWL_SCOPE)
+        self.assertTrue(scope.allows("https://utsabzone.store/products"))
+
+
+INSTALLERS = [
+    "/downloads/linux/amd64/installer/ian-vpn.tar.gz",
+    "/downloads/linux/amd64/installer/ian-vpn.deb",
+    "/downloads/linux/amd64/installer/ian-vpn.rpm",
+]
+TEXT = " ".join(["Our VPN keeps your connection private on every network."] * 5)
+
+
+def vpn_site(request):
+    """A home page that links to a FAQ and to its installers, as ian-vpn.site does."""
+    path = request.url.path
+    if path == "/":
+        links = "".join(f'<a href="{href}">Download</a>' for href in ["/faqs", *INSTALLERS])
+        return httpx.Response(200, html=f"<main><h1>IAN VPN</h1><p>{TEXT}</p>{links}</main>")
+    if path == "/faqs":
+        return httpx.Response(200, html=f"<main><h1>FAQs</h1><p>{TEXT}</p></main>")
+    if path in INSTALLERS:
+        return httpx.Response(200, headers={"content-type": "application/octet-stream"},
+                              content=b"\x7fELF\x00\x00" * 100_000)
+    return httpx.Response(404)
+
+
+class TestDownloadsAreNotPages(unittest.TestCase):
+    """Seen in production: ten links on ian-vpn.site became 149,500 chunks,
+    almost all of them its installers read as text."""
+
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def test_the_pages_are_read_and_the_installers_skipped(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback') as browser:
+            result = reader.crawl("https://ian-vpn.site")
+
+        self.assertEqual(sorted(result), ["https://ian-vpn.site", "https://ian-vpn.site/faqs"])
+        browser.return_value.fetch_with_browser.assert_not_called()
+        # Skipped, not failed: they say nothing about whether the site works
+        self.assertEqual(reader._failure_causes, {})
+        self.assertEqual(reader._failed_crawls, 0)
+
+    def test_the_documents_come_from_the_pages_only(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            documents = reader.read("https://ian-vpn.site")
+
+        self.assertEqual({d.meta_data['url'] for d in documents},
+                         {"https://ian-vpn.site", "https://ian-vpn.site/faqs"})
+        self.assertLess(len(documents), 10)
+
+
+class FakeDeadline:
+    expired = False
+
+
+class TestTheCrawlStopsAtItsDeadline(unittest.TestCase):
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def test_no_new_page_is_read_once_time_is_up(self):
+        deadline = FakeDeadline()
+        requested = []
+
+        def site(request):
+            requested.append(request.url.path)
+            deadline.expired = True  # the home page used the last of the time
+            return vpn_site(request)
+
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50, deadline=deadline)
+        with serving(site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            result = reader.crawl("https://ian-vpn.site")
+
+        self.assertEqual(requested, ["/"])
+        self.assertEqual(list(result), ["https://ian-vpn.site"])
+
+    def test_without_a_deadline_the_crawl_runs_to_its_link_limit(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            result = reader.crawl("https://ian-vpn.site")
+        self.assertEqual(len(result), 2)
+
+
+class TestASeedThatIsNotAPage(unittest.TestCase):
+    """A file given as a website must say so, not blame JavaScript."""
+
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def crawl(self, response):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(lambda request: response), \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            return reader, reader.crawl("https://example.com/brochure.pdf")
+
+    def test_a_file_given_as_the_website_is_named_as_one(self):
+        reader, result = self.crawl(httpx.Response(200, headers={"content-type": "application/pdf"},
+                                                   content=b"%PDF-1.4"))
+        self.assertEqual(result, {})
+        self.assertEqual(reader.dominant_failure_cause(), FailureCause.NOT_A_PAGE)
+        self.assertTrue(FAILURE_ADVICE[FailureCause.NOT_A_PAGE].startswith("{url} is a file"))
+
+    def test_an_oversized_page_is_not_called_a_file(self):
+        reader, _ = self.crawl(httpx.Response(200, headers={"content-type": "text/html",
+                                                            "content-length": str(50 * 1024 * 1024)},
+                                              content=b""))
+        self.assertIsNone(reader.dominant_failure_cause())
+
+    def test_linked_downloads_are_never_a_cause(self):
+        """Only the address given: a site's installer links say nothing about it."""
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            reader.crawl("https://ian-vpn.site")
+        self.assertEqual(reader._failure_causes, {})

backend/tests/knowledge/test_failure_advice.py

@@ -0,0 +1,41 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+A failed source's stored error becomes one plain sentence about "This site".
+"""
+
+import pytest
+
+from app.knowledge.failure_advice import FAILURE_ADVICE, GENERIC_REASON, plain_reason
+
+URL = "https://erfanpv.com"
+
+
+@pytest.mark.parametrize("cause", list(FAILURE_ADVICE))
+def test_our_advice_is_rephrased_about_this_site(cause):
+    reason = plain_reason(FAILURE_ADVICE[cause].format(url=URL), URL)
+    assert reason.startswith("This site ")
+    assert URL not in reason
+
+
+@pytest.mark.parametrize("error,source", [
+    ("Traceback (most recent call last): KeyError 'x'", URL),
+    (FAILURE_ADVICE[next(iter(FAILURE_ADVICE))].format(url="https://other.com"), URL),
+    (None, URL),
+    ("", URL),
+    ("anything", None),
+])
+def test_anything_else_gets_the_generic_reason(error, source):
+    assert plain_reason(error, source) == GENERIC_REASON

backend/tests/knowledge/test_main_content.py

@@ -0,0 +1,132 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from bs4 import BeautifulSoup
+
+from app.knowledge.main_content import content_chars, select_main_node
+
+PROSE = "Verified rooms close to campus with bills included. " * 40
+
+
+def soup_of(html: str) -> BeautifulSoup:
+    return BeautifulSoup(html, "html.parser")
+
+
+def test_a_short_leading_block_does_not_win():
+    """The eazzyliving.co.uk shape, shared by the crawler and the importer.
+
+    Its first <article> was a 108-character search widget. Returning the first
+    candidate over the length floor meant that widget became the whole page and
+    4,500 characters went unindexed.
+    """
+    node = select_main_node(soup_of(f"""
+        <html><body>
+            <article>Search properties. Search by city, university, or
+            neighbourhood to find verified student rooms and studios.</article>
+            <div class="listings"><h1>Rooms in Leeds</h1><p>{PROSE}</p></div>
+        </body></html>
+    """))
+
+    assert node is not None
+    text = node.get_text(" ", strip=True)
+    assert "Rooms in Leeds" in text
+    assert "Verified rooms close to campus" in text
+
+
+def test_prefers_the_specific_container_over_body():
+    """When one container holds the page, index it rather than the whole body.
+
+    Otherwise every page carries the header and footer into the vector store.
+    """
+    node = select_main_node(soup_of(f"""
+        <html><body>
+            <header>Site name. Login. Contact us.</header>
+            <main><h1>Pricing</h1><p>{PROSE}</p></main>
+            <footer>Copyright 2026. Terms. Privacy.</footer>
+        </body></html>
+    """))
+
+    assert node is not None and node.name == "main"
+    assert "Login" not in node.get_text(" ", strip=True)
+
+
+def test_falls_back_to_body_when_no_container_holds_the_page():
+    """Content split across sibling divs belongs to the body, not to one div."""
+    half = "Half of the page's content lives here. " * 20
+    node = select_main_node(soup_of(f"""
+        <html><body>
+            <div class="left"><p>{half}</p></div>
+            <div class="right"><p>{half}</p></div>
+        </body></html>
+    """))
+
+    assert node is not None and node.name == "body"
+
+
+def test_a_page_of_only_chrome_is_reported_as_empty():
+    """www.solcontrol.ca: a phone number and a login link, nothing else.
+
+    Returning None lets the crawler route the page to the browser fallback
+    instead of storing navigation as though it were content.
+    """
+    node = select_main_node(soup_of("""
+        <html><body>
+            <header><a href="/login">Welcome, Guest - Login</a>
+                    <a href="tel:905-230-8468">905-230-8468</a></header>
+            <nav><a href="/products">Products</a><a href="/about">About</a></nav>
+            <footer><a href="mailto:info@solcontrol.ca">info@solcontrol.ca</a></footer>
+        </body></html>
+    """))
+
+    assert node is None
+
+
+def test_scripts_and_styles_never_count_as_content():
+    node = select_main_node(soup_of("""
+        <html><body>
+            <script>var a = 'x'.repeat(5000);</script>
+            <style>.a{color:red}</style>
+            <nav><a href="/x">Home</a></nav>
+        </body></html>
+    """))
+    assert node is None
+
+
+def test_content_chars_ignores_navigation_chrome():
+    body = soup_of("""
+        <html><body>
+            <nav>Products About Contact Careers Support Blog</nav>
+            <p>Real content.</p>
+        </body></html>
+    """).find("body")
+
+    assert content_chars(body) == len("Real content.")
+
+
+def test_nested_chrome_is_only_subtracted_once():
+    """A <nav> inside a <header> must not be discounted twice.
+
+    Double-subtracting understates the page and can push a real page below the
+    threshold, where it would be reported as having no content at all.
+    """
+    body = soup_of("""
+        <html><body>
+            <header>Site<nav>Home About</nav></header>
+            <p>Real content.</p>
+        </body></html>
+    """).find("body")
+
+    assert content_chars(body) == len("Real content.")

backend/tests/knowledge/test_optimized_pgvector.py

@@ -0,0 +1,205 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import pytest
+from unittest.mock import MagicMock, patch
+import numpy as np
+
+from agno.document import Document
+from agno.vectordb.pgvector.pgvector import PgVector
+from app.knowledge.optimized_pgvector import OptimizedPgVector
+
+
+class TestOptimizedPgVector:
+    """Test cases for OptimizedPgVector"""
+    
+    @pytest.fixture
+    def mock_embedder(self):
+        """Create a mock embedder"""
+        mock = MagicMock()
+        mock.get_embedding.return_value = np.array([0.1, 0.2, 0.3])
+        mock.dimensions = 3
+        return mock
+    
+    @pytest.fixture
+    def mock_session(self):
+        """Create a mock database session"""
+        mock = MagicMock()
+        # Configure session to return a context manager
+        mock.return_value.__enter__.return_value = mock
+        return mock
+    
+    @patch('sqlalchemy.engine.create_engine')
+    def test_init(self, mock_create_engine, mock_embedder):
+        """Test initialization"""
+        # Arrange
+        mock_engine = MagicMock()
+        mock_create_engine.return_value = mock_engine
+        
+        # Act
+        db = OptimizedPgVector(
+            table_name="test_table",
+            db_url="postgresql+psycopg://user:pass@localhost/testdb",
+            embedder=mock_embedder
+        )
+        
+        # Assert
+        assert db.table_name == "test_table"
+        assert db.embedder == mock_embedder
+        assert db.dimensions == 3
+    
+    @patch('sqlalchemy.engine.create_engine')
+    def test_upsert_with_pre_embedded_documents(self, mock_create_engine, mock_embedder, mock_session):
+        """Test upserting pre-embedded documents"""
+        # Arrange
+        mock_engine = MagicMock()
+        mock_create_engine.return_value = mock_engine
+        
+        db = OptimizedPgVector(
+            table_name="test_table",
+            db_url="postgresql+psycopg://user:pass@localhost/testdb",
+            embedder=mock_embedder
+        )
+        db.Session = mock_session
+        db._clean_content = lambda x: x  # Mock the content cleaning function
+        
+        # Create a document with an existing embedding
+        pre_embedded_doc = Document(
+            id="test1",
+            name="Test Document 1",
+            content="This is a test document",
+            embedding=np.array([0.5, 0.6, 0.7])
+        )
+        
+        # Create a document without an embedding
+        unembedded_doc = Document(
+            id="test2",
+            name="Test Document 2",
+            content="This is another test document"
+        )
+        
+        # Mock the document's embed method
+        with patch.object(Document, 'embed') as mock_embed:
+            # Act
+            with patch('sqlalchemy.dialects.postgresql.insert') as mock_insert:
+                mock_insert.return_value.on_conflict_do_update.return_value = "upsert_stmt"
+                db.upsert([pre_embedded_doc, unembedded_doc])
+            
+            # Assert
+            # The document's embed method should be called for the unembedded document
+            mock_embed.assert_called_once_with(embedder=mock_embedder)
+            
+            # Both documents should be passed to the upsert statement
+            assert mock_session.execute.call_count == 1
+        
+    @patch('sqlalchemy.engine.create_engine')
+    def test_insert_with_pre_embedded_documents(self, mock_create_engine, mock_embedder, mock_session):
+        """Test inserting pre-embedded documents"""
+        # Arrange
+        mock_engine = MagicMock()
+        mock_create_engine.return_value = mock_engine
+        
+        db = OptimizedPgVector(
+            table_name="test_table",
+            db_url="postgresql+psycopg://user:pass@localhost/testdb",
+            embedder=mock_embedder
+        )
+        db.Session = mock_session
+        db._clean_content = lambda x: x  # Mock the content cleaning function
+        
+        # Create a document with an existing embedding
+        pre_embedded_doc = Document(
+            id="test1",
+            name="Test Document 1",
+            content="This is a test document",
+            embedding=np.array([0.5, 0.6, 0.7])
+        )
+        
+        # Create a document without an embedding
+        unembedded_doc = Document(
+            id="test2",
+            name="Test Document 2",
+            content="This is another test document"
+        )
+        
+        # Mock the document's embed method
+        with patch.object(Document, 'embed') as mock_embed:
+            # Act
+            with patch('sqlalchemy.dialects.postgresql.insert') as mock_insert:
+                db.insert([pre_embedded_doc, unembedded_doc])
+            
+            # Assert
+            # The document's embed method should be called for the unembedded document
+            mock_embed.assert_called_once_with(embedder=mock_embedder)
+            
+            # Both documents should be passed to the insert statement
+            assert mock_session.execute.call_count == 1
+
+
+if __name__ == "__main__":
+    pytest.main(["-v", "test_optimized_pgvector.py"])
+
+
+class _PgError(Exception):
+    """Stand-in for a psycopg error, which carries the SQLSTATE as .sqlstate."""
+
+    def __init__(self, sqlstate):
+        super().__init__(sqlstate)
+        self.sqlstate = sqlstate
+
+
+class TestConcurrentTableCreate:
+    """The per-org vector table is created by whichever queue item gets there first.
+
+    agno's PgVector.create() does table_exists() and then CREATE TABLE, which is
+    not atomic. The processor runs two items at once, so an organisation whose
+    first two sources are queued together had both workers create the table and
+    the loser died with
+    `duplicate key value violates unique constraint "pg_type_typname_nsp_index"`,
+    losing that source (production queue item 372, goldmaidsteam.com).
+    """
+
+    def _vector_db(self):
+        return OptimizedPgVector.__new__(OptimizedPgVector)
+
+    @pytest.mark.parametrize("sqlstate", ["42P07", "23505"])
+    def test_losing_the_create_race_is_not_an_error(self, sqlstate):
+        db = self._vector_db()
+        db.table_name = "d_org"
+        db.schema = "ai"
+
+        with patch.object(PgVector, "create", side_effect=_PgError(sqlstate)):
+            db.create()  # the other worker created it; that is the table we wanted
+
+    def test_a_real_create_failure_still_raises(self):
+        db = self._vector_db()
+        db.table_name = "d_org"
+        db.schema = "ai"
+
+        with patch.object(PgVector, "create", side_effect=_PgError("42501")):
+            with pytest.raises(_PgError):
+                db.create()
+
+    def test_duplicate_detected_through_sqlalchemy_wrapping(self):
+        """SQLAlchemy wraps the driver error; the SQLSTATE lives on .orig."""
+        wrapped = RuntimeError("duplicate")
+        wrapped.orig = _PgError("23505")
+
+        with patch.object(PgVector, "create", side_effect=wrapped):
+            db = self._vector_db()
+            db.table_name = "d_org"
+            db.schema = "ai"
+            db.create()

backend/tests/knowledge/test_page_fetch.py

@@ -0,0 +1,137 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+A crawl reads pages, not the downloads they link to. Seen in production:
+ian-vpn.site linked its Linux installers, each one read as text became
+~30,000 chunks, and the worker hit its memory limit three hours later.
+"""
+
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+from app.knowledge.page_fetch import MAX_PAGE_BYTES, NotAPage, fetch_page_text
+
+URL = "https://ian-vpn.site/downloads/linux/amd64/installer/ian-vpn.tar.gz"
+
+
+@pytest.fixture(autouse=True)
+def public_dns():
+    with patch("app.knowledge.url_safety.socket.getaddrinfo",
+               return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
+        yield
+
+
+class Unread(httpx.SyncByteStream):
+    """A body that must never be downloaded."""
+
+    def __iter__(self):
+        raise AssertionError("the body of a download was read")
+
+
+class Counted(httpx.SyncByteStream):
+    """`total` bytes in 64 KB chunks, counting what was read."""
+
+    def __init__(self, total: int, first: bytes = b""):
+        self.total, self.first, self.read = total, first, 0
+
+    def __iter__(self):
+        chunk = 64 * 1024
+        while self.read < self.total:
+            piece = (self.first if self.read == 0 and self.first else b"a" * chunk)[:self.total - self.read]
+            self.read += len(piece)
+            yield piece
+
+
+def fetch(response: httpx.Response, **kwargs) -> str:
+    with httpx.Client(transport=httpx.MockTransport(lambda request: response)) as client:
+        return fetch_page_text(client, URL, **kwargs)
+
+
+def test_a_page_is_read():
+    assert "Hello" in fetch(httpx.Response(200, html="<p>Hello</p>"))
+
+
+def test_plain_text_is_a_page_too():
+    assert fetch(httpx.Response(200, text="Plain notes")) == "Plain notes"
+
+
+@pytest.mark.parametrize("kind", [
+    "application/gzip", "application/x-tar", "application/octet-stream",
+    "application/vnd.debian.binary-package", "application/x-rpm",
+    "application/zip", "application/pdf", "image/png", "video/mp4",
+])
+def test_a_download_is_skipped_without_reading_it(kind):
+    with pytest.raises(NotAPage, match="download"):
+        fetch(httpx.Response(200, headers={"content-type": kind}, stream=Unread()))
+
+
+def test_a_download_sent_without_a_type_is_recognised_by_its_bytes():
+    binary = Counted(10 * 1024 * 1024, first=b"\x1f\x8b\x08\x00" + b"\x00" * 100)
+    with pytest.raises(NotAPage, match="binary"):
+        fetch(httpx.Response(200, stream=binary))
+    assert binary.read < 1024 * 1024  # stopped at the first chunk
+
+
+def test_a_page_sent_without_a_type_is_still_read():
+    assert fetch(httpx.Response(200, content=b"<p>Untyped page</p>")) == "<p>Untyped page</p>"
+
+
+def test_a_declared_oversized_body_is_skipped_without_reading_it():
+    big = httpx.Response(200, headers={"content-type": "text/html",
+                                       "content-length": str(MAX_PAGE_BYTES + 1)}, stream=Unread())
+    with pytest.raises(NotAPage, match="larger"):
+        fetch(big)
+
+
+def test_reading_stops_at_the_cap_when_the_size_is_not_declared():
+    endless = Counted(50 * 1024 * 1024)
+    with pytest.raises(NotAPage, match="larger"):
+        fetch(httpx.Response(200, headers={"content-type": "text/html"}, stream=endless), max_bytes=1024 * 1024)
+    assert endless.read <= 1024 * 1024 + 64 * 1024
+
+
+def test_the_declared_charset_is_used():
+    body = "<p>Café</p>".encode("latin-1")
+    assert fetch(httpx.Response(200, headers={"content-type": "text/html; charset=iso-8859-1"},
+                                content=body)) == "<p>Café</p>"
+
+
+def test_an_error_answer_is_still_an_error():
+    with pytest.raises(httpx.HTTPStatusError):
+        fetch(httpx.Response(404, html="<p>Not found</p>"))
+
+
+def test_a_utf16_page_is_read_not_taken_for_a_binary():
+    page = "<p>Hello from a UTF-16 page</p>"
+    assert fetch(httpx.Response(200, headers={"content-type": "text/html; charset=utf-16"},
+                                content=page.encode("utf-16"))) == page
+
+
+@pytest.mark.parametrize("kind", ["application/xml", "text/xml"])
+def test_xhtml_served_as_xml_is_a_page(kind):
+    assert "Docs" in fetch(httpx.Response(200, headers={"content-type": kind},
+                                          content=b"<html><body><p>Docs</p></body></html>"))
+
+
+def test_only_a_download_is_called_one():
+    with pytest.raises(NotAPage) as download:
+        fetch(httpx.Response(200, headers={"content-type": "application/pdf"}, stream=Unread()))
+    big = httpx.Response(200, headers={"content-type": "text/html", "content-length": str(MAX_PAGE_BYTES + 1)},
+                         stream=Unread())
+    with pytest.raises(NotAPage) as too_big:
+        fetch(big)
+    assert (download.value.download, too_big.value.download) == (True, False)

backend/tests/knowledge/test_pdf_download.py

@@ -0,0 +1,122 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+from app.knowledge import pdf_download, url_safety
+from app.knowledge.pdf_download import PdfTooLargeError, download_pdf
+from app.knowledge.url_safety import BlockedHostError
+
+PDF = b"%PDF-1.4 test body"
+PUBLIC_IP = [(2, 1, 6, "", ("93.184.216.34", 0))]
+
+
+@pytest.fixture
+def served(monkeypatch):
+    """Route download_pdf's httpx client to an in-process handler and record
+    every URL it was asked to open."""
+    requested = []
+    routes = {}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        url = str(request.url)
+        requested.append(url)
+        return routes[url]()
+
+    real_client = httpx.Client
+    monkeypatch.setattr(
+        pdf_download.httpx, "Client",
+        lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs),
+    )
+    return routes, requested
+
+
+@pytest.fixture
+def public_dns():
+    with patch.object(url_safety.socket, "getaddrinfo", return_value=PUBLIC_IP):
+        yield
+
+
+@pytest.mark.parametrize("url", [
+    "http://127.0.0.1/internal.pdf",
+    "http://localhost:8000/api/v1/secret.pdf",
+    "http://169.254.169.254/latest/meta-data/",
+])
+def test_internal_url_is_refused_before_any_request(served, tmp_path, url):
+    _, requested = served
+    with pytest.raises(BlockedHostError):
+        download_pdf(url, str(tmp_path / "out.pdf"))
+    assert requested == []
+
+
+def test_hostname_resolving_to_internal_address_is_refused(served, tmp_path):
+    _, requested = served
+    with patch.object(url_safety.socket, "getaddrinfo",
+                      return_value=[(2, 1, 6, "", ("10.0.0.7", 0))]):
+        with pytest.raises(BlockedHostError):
+            download_pdf("https://pdfs.attacker.example/a.pdf", str(tmp_path / "out.pdf"))
+    assert requested == []
+
+
+def test_public_url_redirecting_to_private_address_is_refused(served, public_dns, tmp_path):
+    routes, requested = served
+    routes["https://files.example.com/doc.pdf"] = lambda: httpx.Response(
+        302, headers={"location": "http://169.254.169.254/latest/meta-data/"})
+    with pytest.raises(BlockedHostError):
+        download_pdf("https://files.example.com/doc.pdf", str(tmp_path / "out.pdf"))
+    # The internal hop was never opened.
+    assert requested == ["https://files.example.com/doc.pdf"]
+
+
+def test_public_url_is_streamed_to_disk_following_safe_redirects(served, public_dns, tmp_path):
+    routes, _ = served
+    routes["https://files.example.com/doc.pdf"] = lambda: httpx.Response(
+        301, headers={"location": "/v2/doc.pdf"})
+    routes["https://files.example.com/v2/doc.pdf"] = lambda: httpx.Response(200, content=PDF)
+    dest = tmp_path / "out.pdf"
+
+    assert download_pdf("https://files.example.com/doc.pdf", str(dest)) == str(dest)
+    assert dest.read_bytes() == PDF
+
+
+def test_declared_oversize_is_refused(served, public_dns, tmp_path):
+    routes, _ = served
+    routes["https://files.example.com/big.pdf"] = lambda: httpx.Response(
+        200, content=PDF, headers={"content-length": str(10 ** 9)})
+    with pytest.raises(PdfTooLargeError):
+        download_pdf("https://files.example.com/big.pdf", str(tmp_path / "out.pdf"))
+
+
+def test_body_past_the_cap_is_refused_even_without_content_length(served, public_dns, tmp_path):
+    routes, _ = served
+
+    def chunked():
+        return httpx.Response(200, content=iter([PDF] * 10))  # streamed: no length
+
+    routes["https://files.example.com/big.pdf"] = chunked
+    with pytest.raises(PdfTooLargeError):
+        download_pdf("https://files.example.com/big.pdf", str(tmp_path / "out.pdf"),
+                     max_bytes=len(PDF) * 2)
+
+
+def test_http_error_is_raised(served, public_dns, tmp_path):
+    routes, _ = served
+    routes["https://files.example.com/gone.pdf"] = lambda: httpx.Response(404)
+    with pytest.raises(httpx.HTTPStatusError):
+        download_pdf("https://files.example.com/gone.pdf", str(tmp_path / "out.pdf"))

backend/tests/knowledge/test_pdf_limits.py

@@ -0,0 +1,75 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+A PDF's cost is its pages, not its bytes: each page is embedded, and a
+scanned one OCR'd. Past the page limit it is refused before any is read.
+"""
+
+import io
+from unittest.mock import patch
+
+import pytest
+from pypdf import PdfWriter
+
+from app.core.config import settings
+from app.knowledge.pdf_download import PdfTooLargeError
+from app.knowledge.pdf_limits import PdfRefused, PdfTooManyPages, check_page_count, count_pages
+
+
+def pdf_with_pages(n: int) -> bytes:
+    writer = PdfWriter()
+    for _ in range(n):
+        writer.add_blank_page(width=200, height=200)
+    out = io.BytesIO()
+    writer.write(out)
+    return out.getvalue()
+
+
+def test_the_pages_are_counted():
+    assert count_pages(io.BytesIO(pdf_with_pages(7))) == 7
+
+
+def test_a_file_path_is_counted_too(tmp_path):
+    path = tmp_path / "manual.pdf"
+    path.write_bytes(pdf_with_pages(4))
+    assert count_pages(str(path)) == 4
+
+
+def test_a_pdf_at_the_limit_is_accepted():
+    check_page_count(io.BytesIO(pdf_with_pages(3)), max_pages=3)
+
+
+def test_a_pdf_over_the_limit_is_refused_and_told_why():
+    with pytest.raises(PdfTooManyPages) as refused:
+        check_page_count(io.BytesIO(pdf_with_pages(4)), max_pages=3)
+    assert str(refused.value) == ("This PDF has 4 pages; the limit is 3. Upload the part you need, "
+                                  "or split it into smaller files.")
+
+
+def test_the_limit_comes_from_settings():
+    with patch.object(settings, "KB_PDF_MAX_PAGES", 2), pytest.raises(PdfTooManyPages):
+        check_page_count(io.BytesIO(pdf_with_pages(3)))
+
+
+@pytest.mark.parametrize("damaged", [b"%PDF-1.4 test pdf content", b"", b"not a pdf"])
+def test_one_that_cannot_be_counted_is_left_to_the_reader(damaged):
+    assert count_pages(io.BytesIO(damaged)) is None
+    check_page_count(io.BytesIO(damaged), max_pages=1)  # no refusal: the reader decides
+
+
+def test_both_limits_are_refusals_with_a_message_for_the_user():
+    too_big = PdfTooLargeError(25 * 1024 * 1024)
+    assert isinstance(too_big, PdfRefused)
+    assert str(too_big) == "This PDF is larger than 25 MB. Upload a smaller file, or split it into parts."

backend/tests/knowledge/test_rechunk_knowledge.py

@@ -0,0 +1,149 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+from scripts import rechunk_knowledge as script
+
+
+def _row(id: str, content: str, **meta):
+    return SimpleNamespace(
+        id=id,
+        name="https://x.com",
+        content=content,
+        meta_data={"url": id, **meta},
+        filters={"name": "https://x.com", "agent_id": ["a1"], "org_id": "o1"},
+    )
+
+
+@pytest.fixture
+def conn():
+    return MagicMock()
+
+
+@pytest.fixture
+def embedder():
+    embedder = MagicMock()
+    embedder.get_embedding_and_usage.return_value = ([0.5, 0.5], None)
+    return embedder
+
+
+def _execute_returning(conn, oversized, taken=()):
+    """First query lists oversized rows, second checks planned ids."""
+    results = [MagicMock(fetchall=MagicMock(return_value=oversized)),
+               MagicMock(fetchall=MagicMock(return_value=[SimpleNamespace(id=t) for t in taken]))]
+    results += [MagicMock()] * 10
+    conn.execute.side_effect = results
+
+
+class TestProcessTable:
+    def test_splits_row_and_replaces_it_then_commits(self, conn, embedder):
+        big = _row("https://x.com/pricing", "para\n\n" * 400)
+        _execute_returning(conn, [big])
+
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+
+        assert report.rows == 1 and report.failed == 0 and report.chunks > 1
+        # Commit-as-you-go: never conn.begin() (the preceding SELECTs already
+        # opened the transaction), one commit after insert + delete.
+        conn.begin.assert_not_called()
+        conn.commit.assert_called_once()
+        insert_call, delete_call = conn.execute.call_args_list[2], conn.execute.call_args_list[3]
+        records = insert_call.args[1]
+        assert len(records) == report.chunks
+        assert records[0]["id"] == "https://x.com/pricing::1"
+        assert '"agent_id": ["a1"]' in records[0]["filters"]
+        assert '"page_id": "https://x.com/pricing"' in records[0]["meta_data"]
+        assert delete_call.args[1] == {"id": "https://x.com/pricing"}
+        assert embedder.get_embedding_and_usage.call_count == report.chunks
+
+    def test_dry_run_writes_nothing(self, conn, embedder):
+        _execute_returning(conn, [_row("https://x.com/p", "x " * 400)])
+        report = script.process_table(conn, "d_o1", None, size=300, dry_run=True)
+        assert report.rows == 1 and report.chunks > 1
+        conn.commit.assert_not_called()
+        assert conn.execute.call_count == 2  # list + collision check only
+
+    def test_collision_leaves_row_untouched(self, conn, embedder):
+        _execute_returning(conn, [_row("https://x.com/p", "x " * 400)], taken=["https://x.com/p::2"])
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+        assert report.failed == 1 and report.collisions == ["https://x.com/p::2"]
+        conn.commit.assert_not_called()
+
+    def test_embedding_failure_keeps_original(self, conn, embedder):
+        _execute_returning(conn, [_row("https://x.com/p", "x " * 400)])
+        embedder.get_embedding_and_usage.side_effect = RuntimeError("model down")
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+        assert report.failed == 1
+        conn.commit.assert_not_called()
+
+    def test_write_failure_rolls_back(self, conn, embedder):
+        _execute_returning(conn, [_row("https://x.com/p", "x " * 400)])
+        conn.execute.side_effect = list(conn.execute.side_effect)[:2] + [RuntimeError("disk full")]
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+        assert report.failed == 1
+        conn.rollback.assert_called_once()
+        conn.commit.assert_not_called()
+
+    def test_row_that_fits_one_chunk_is_skipped(self, conn, embedder):
+        _execute_returning(conn, [_row("https://x.com/p", "short")])
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+        assert report.skipped == 1 and report.rows == 0
+
+    def test_legacy_pdf_chunks_are_left_alone(self, conn, embedder):
+        pdf = SimpleNamespace(id="manual_3", name="manual", content="p " * 400,
+                              meta_data={"page": 3}, filters=None)
+        _execute_returning(conn, [pdf])
+        report = script.process_table(conn, "d_o1", embedder, size=300, dry_run=False)
+        assert report.legacy_pdf == 1 and report.rows == 0
+        conn.commit.assert_not_called()
+
+
+class TestRenameLegacySuffixedPages:
+    def test_renames_to_explicit_chunk_id(self, conn):
+        conn.execute.return_value.fetchall.return_value = [
+            SimpleNamespace(id="https://x.com/post_2", meta_data={"url": "https://x.com/post_2"})
+        ]
+        renamed = script.rename_legacy_suffixed_pages(conn, "d_o1", size=300, dry_run=False)
+        assert renamed == 1
+        params = conn.execute.call_args.args[1]
+        assert params["new_id"] == "https://x.com/post_2::1"
+        assert params["old_id"] == "https://x.com/post_2"
+        assert '"page_id": "https://x.com/post_2"' in params["meta"]
+        conn.commit.assert_called_once()
+
+    def test_dry_run_only_reports(self, conn):
+        conn.execute.return_value.fetchall.return_value = [
+            SimpleNamespace(id="https://x.com/post_2", meta_data={})
+        ]
+        assert script.rename_legacy_suffixed_pages(conn, "d_o1", size=300, dry_run=True) == 1
+        assert conn.execute.call_count == 1
+        conn.commit.assert_not_called()
+
+
+class TestVectorTables:
+    def test_org_scopes_to_one_table(self, conn):
+        assert script.vector_tables(conn, "abc") == ["d_abc"]
+        conn.execute.assert_not_called()
+
+    def test_lists_all_vector_tables(self, conn):
+        conn.execute.return_value.fetchall.return_value = [
+            SimpleNamespace(table_name="d_1"), SimpleNamespace(table_name="d_2")
+        ]
+        assert script.vector_tables(conn, None) == ["d_1", "d_2"]

backend/tests/knowledge/test_rechunk_knowledge_transaction.py

@@ -0,0 +1,67 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+scripts.rechunk_knowledge against a real SQLAlchemy connection.
+
+The other script tests mock the connection, which hides transaction
+semantics: SQLAlchemy 2.x autobegins a transaction on the first SELECT, and
+``Connection.begin()`` then raises ``InvalidRequestError`` until that
+transaction is committed or rolled back. ``process_table`` runs two SELECTs
+before ``replace_row`` calls ``conn.begin()``, so every row fails.
+"""
+
+import hashlib
+from unittest.mock import MagicMock, patch
+
+import pytest
+from sqlalchemy import create_engine, event, text
+
+from scripts import rechunk_knowledge as script
+
+
+@pytest.fixture
+def conn():
+    engine = create_engine("sqlite://")
+
+    @event.listens_for(engine, "connect")
+    def _setup(dbapi_conn, _):
+        dbapi_conn.create_function("md5", 1, lambda s: hashlib.md5(s.encode()).hexdigest())
+        dbapi_conn.execute("ATTACH DATABASE ':memory:' AS ai")
+        dbapi_conn.execute(
+            'CREATE TABLE ai."d_o" (id TEXT PRIMARY KEY, name TEXT, meta_data TEXT, filters TEXT, '
+            "content TEXT, embedding TEXT, usage TEXT, content_hash TEXT)"
+        )
+
+    with engine.connect() as conn:
+        conn.execute(
+            text('INSERT INTO ai."d_o" (id, name, content, meta_data, filters) VALUES (:id, :n, :c, NULL, NULL)'),
+            {"id": "https://x.com/p", "n": "https://x.com", "c": "para. " * 300},
+        )
+        conn.commit()
+        yield conn
+
+
+def test_process_table_replaces_row_on_a_real_connection(conn):
+    embedder = MagicMock()
+    embedder.get_embedding_and_usage.return_value = ([0.5, 0.5], None)
+
+    # ``id = ANY(:ids)`` is Postgres-only; the collision check is not what is under test.
+    with patch.object(script, "existing_ids", lambda *_: []):
+        report = script.process_table(conn, "d_o", embedder, size=300, dry_run=False)
+
+    assert report.failed == 0, "replace_row must work after the SELECTs that precede it"
+    ids = [r.id for r in conn.execute(text('SELECT id FROM ai."d_o" ORDER BY length(id), id')).fetchall()]
+    assert ids and ids[0] == "https://x.com/p::1" and "https://x.com/p" not in ids
+    assert len(ids) == report.chunks

backend/tests/knowledge/test_sitemap_parser.py

@@ -0,0 +1,150 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import gzip
+from unittest.mock import patch
+
+from app.knowledge import sitemap_parser
+
+NS = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'
+
+URLSET = f"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset {NS}>
+  <url><loc>https://site.com/a</loc></url>
+  <url><loc>https://site.com/b</loc></url>
+  <url><loc>https://site.com/a</loc></url>
+</urlset>""".encode()
+
+INDEX = f"""<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex {NS}>
+  <sitemap><loc>https://site.com/sitemap-1.xml</loc></sitemap>
+  <sitemap><loc>https://site.com/sitemap-2.xml</loc></sitemap>
+</sitemapindex>""".encode()
+
+CHILD1 = f"""<urlset {NS}><url><loc>https://site.com/1</loc></url></urlset>""".encode()
+CHILD2 = f"""<urlset {NS}><url><loc>https://site.com/2</loc></url></urlset>""".encode()
+
+
+def _fake_fetch(mapping):
+    """Return a _fetch replacement that serves bytes from a url->bytes mapping."""
+    def _fetch(client, url):
+        return mapping.get(url)
+    return _fetch
+
+
+def test_parse_urlset_dedupes_and_preserves_order():
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch({"https://site.com/sitemap.xml": URLSET})):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/a", "https://site.com/b"]
+
+
+def test_sitemap_index_recurses_into_children():
+    mapping = {
+        "https://site.com/sitemap.xml": INDEX,
+        "https://site.com/sitemap-1.xml": CHILD1,
+        "https://site.com/sitemap-2.xml": CHILD2,
+    }
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch(mapping)):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/1", "https://site.com/2"]
+
+
+def test_max_urls_caps_the_result():
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch({"https://site.com/sitemap.xml": URLSET})):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=1)
+    assert urls == ["https://site.com/a"]
+
+
+def test_gzip_content_without_extension_is_parsed():
+    gzipped = gzip.compress(URLSET)
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch({"https://site.com/sitemap.xml": gzipped})):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/a", "https://site.com/b"]
+
+
+def test_child_fetch_failure_is_skipped():
+    mapping = {
+        "https://site.com/sitemap.xml": INDEX,
+        "https://site.com/sitemap-1.xml": CHILD1,
+        # sitemap-2.xml returns None (fetch failure)
+    }
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch(mapping)):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/1"]
+
+
+def test_invalid_xml_returns_empty():
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch({"https://site.com/sitemap.xml": b"not xml <<<"})):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == []
+
+
+def test_cross_domain_child_sitemap_is_skipped():
+    evil_index = f"""<sitemapindex {NS}>
+      <sitemap><loc>https://site.com/ok.xml</loc></sitemap>
+      <sitemap><loc>https://evil.com/steal.xml</loc></sitemap>
+    </sitemapindex>""".encode()
+    mapping = {
+        "https://site.com/sitemap.xml": evil_index,
+        "https://site.com/ok.xml": CHILD1,
+        "https://evil.com/steal.xml": CHILD2,  # must NOT be fetched
+    }
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch(mapping)):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/1"]  # evil.com child skipped
+
+
+def test_cross_domain_page_loc_is_skipped():
+    """A page <loc> on another registrable domain is dropped, so one source
+    can't pull in other domains' URLs. Subdomains of the root are kept."""
+    mixed = f"""<urlset {NS}>
+      <url><loc>https://site.com/a</loc></url>
+      <url><loc>https://blog.site.com/b</loc></url>
+      <url><loc>https://evilsite.com/c</loc></url>
+      <url><loc>https://other.com/d</loc></url>
+    </urlset>""".encode()
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch({"https://site.com/sitemap.xml": mixed})):
+        urls = sitemap_parser.fetch_sitemap_urls("https://site.com/sitemap.xml", max_urls=50)
+    assert urls == ["https://site.com/a", "https://blog.site.com/b"]
+
+
+def test_bounded_gunzip_rejects_oversize(monkeypatch):
+    monkeypatch.setattr(sitemap_parser, "MAX_DECOMPRESSED_BYTES", 100)
+    bomb = gzip.compress(b"A" * 5000)  # decompresses to 5000 > 100
+    assert sitemap_parser._bounded_gunzip(bomb) is None
+    small = gzip.compress(b"A" * 50)
+    assert sitemap_parser._bounded_gunzip(small) == b"A" * 50
+
+
+def test_max_sitemaps_bounds_index_fanout():
+    # A sitemap index pointing at many children; only max_sitemaps files are fetched
+    # (1 index + 1 child when max_sitemaps=2).
+    big_index = (
+        f'<sitemapindex {NS}>'
+        + "".join(f"<sitemap><loc>https://site.com/s{i}.xml</loc></sitemap>" for i in range(10))
+        + "</sitemapindex>"
+    ).encode()
+    mapping = {"https://site.com/sitemap.xml": big_index}
+    for i in range(10):
+        mapping[f"https://site.com/s{i}.xml"] = (
+            f"<urlset {NS}><url><loc>https://site.com/p{i}</loc></url></urlset>".encode()
+        )
+    with patch.object(sitemap_parser, "_fetch", _fake_fetch(mapping)):
+        urls = sitemap_parser.fetch_sitemap_urls(
+            "https://site.com/sitemap.xml", max_urls=50, max_sitemaps=2
+        )
+    # index + first child only
+    assert urls == ["https://site.com/p0"]

backend/tests/knowledge/test_sitemap_reader.py

@@ -0,0 +1,81 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from unittest.mock import patch
+
+from app.knowledge.sitemap_reader import SitemapReader
+
+SITEMAP = "https://site.com/sitemap.xml"
+
+
+def test_crawl_uses_sitemap_pages_and_caps_at_max_links():
+    reader = SitemapReader(max_depth=1, max_links=2, min_content_length=10)
+    pages = ["https://site.com/a", "https://site.com/b", "https://site.com/c"]
+    collected = []
+
+    with patch(
+        "app.knowledge.sitemap_reader.fetch_sitemap_urls", return_value=pages
+    ) as mock_parse, patch.object(
+        reader, "_process_url", side_effect=lambda info, domain: (info[0], f"content {info[0]}", [])
+    ):
+        result = reader.crawl(SITEMAP, on_document_callback=lambda u, c: collected.append(u))
+
+    # The parser is asked for at most max_links pages.
+    assert mock_parse.call_args.kwargs["max_urls"] == 2
+    # _run_crawl_queue caps stored pages at max_links even if more were seeded.
+    assert len(result) == 2
+    assert len(collected) == 2
+    assert all(url in pages for url in result)
+
+
+def test_crawl_empty_sitemap_returns_empty():
+    reader = SitemapReader(max_depth=1, max_links=5)
+    with patch("app.knowledge.sitemap_reader.fetch_sitemap_urls", return_value=[]):
+        result = reader.crawl(SITEMAP)
+    assert result == {}
+
+
+def test_document_groups_under_sitemap_name_with_page_id():
+    # Every discovered page is one sub-page of the single sitemap source:
+    # document.name = sitemap url (the group), document.id = page url.
+    reader = SitemapReader(max_depth=1, max_links=5)
+    doc = reader._create_document_from_content(
+        "https://site.com/pricing", "Plans and pricing content", SITEMAP, 1
+    )
+    assert doc.id == "https://site.com/pricing"
+    assert doc.name == SITEMAP
+
+
+def test_sitemap_page_count_signals_empty_vs_found():
+    reader = SitemapReader(max_depth=1, max_links=5)
+    with patch("app.knowledge.sitemap_reader.fetch_sitemap_urls", return_value=[]):
+        reader.crawl(SITEMAP)
+    assert reader._sitemap_page_count == 0  # dispatcher fails the source on this
+
+    with patch(
+        "app.knowledge.sitemap_reader.fetch_sitemap_urls",
+        return_value=["https://site.com/a", "https://site.com/b"],
+    ), patch.object(reader, "_process_url", side_effect=lambda info, domain: (info[0], "c", [])):
+        reader.crawl(SITEMAP)
+    assert reader._sitemap_page_count == 2
+
+
+def test_crawl_resets_challenge_counter():
+    reader = SitemapReader(max_depth=1, max_links=5)
+    reader._challenge_blocked = 7  # stale from a previous run
+    with patch("app.knowledge.sitemap_reader.fetch_sitemap_urls", return_value=[]):
+        reader.crawl(SITEMAP)
+    assert reader._challenge_blocked == 0

backend/tests/knowledge/test_url_safety.py

@@ -0,0 +1,125 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import ipaddress
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from app.knowledge import url_safety
+from app.knowledge.url_safety import (
+    BlockedHostError,
+    guard_url,
+    is_blocked_host,
+    is_public_ip,
+    resolves_to_blocked_host,
+    safe_get,
+)
+
+
+def test_is_blocked_host_literal_ips():
+    assert is_blocked_host("http://169.254.169.254/latest/meta-data/")  # cloud metadata
+    assert is_blocked_host("http://127.0.0.1/x")
+    assert is_blocked_host("http://10.0.0.5/x")
+    assert is_blocked_host("http://192.168.1.1/x")
+    assert is_blocked_host("http://localhost/x")
+    assert not is_blocked_host("https://site.com/x")
+    assert not is_blocked_host("https://8.8.8.8/x")  # public IP allowed
+
+
+def test_resolves_to_blocked_host_when_hostname_maps_to_internal_ip():
+    # Attacker domain whose A record points at the metadata IP (DNS rebinding).
+    with patch.object(
+        url_safety.socket, "getaddrinfo",
+        return_value=[(2, 1, 6, "", ("169.254.169.254", 0))],
+    ):
+        assert resolves_to_blocked_host("http://imds.attacker.com/latest/meta-data/")
+
+
+def test_resolves_to_blocked_host_allows_public_resolution():
+    with patch.object(
+        url_safety.socket, "getaddrinfo",
+        return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
+    ):
+        assert not resolves_to_blocked_host("https://example.com/x")
+
+
+def test_resolves_to_blocked_host_unresolvable_is_not_blocked():
+    import socket as _socket
+    with patch.object(url_safety.socket, "getaddrinfo", side_effect=_socket.gaierror):
+        assert not resolves_to_blocked_host("https://nope.invalid/x")
+
+
+def _redirect_response(location, url="https://site.com/start"):
+    resp = MagicMock()
+    resp.is_redirect = True
+    resp.headers = {"location": location}
+    resp.url = url
+    return resp
+
+
+def _ok_response():
+    resp = MagicMock()
+    resp.is_redirect = False
+    return resp
+
+
+def test_safe_get_blocks_redirect_to_internal_ip():
+    # A public URL 302-redirects to the metadata IP; safe_get must refuse the hop.
+    client = MagicMock()
+    client.get.return_value = _redirect_response("http://169.254.169.254/")
+    with patch.object(url_safety, "resolves_to_blocked_host", side_effect=lambda u: "169.254" in u):
+        with pytest.raises(BlockedHostError):
+            safe_get(client, "https://site.com/start")
+
+
+def test_safe_get_follows_safe_redirect():
+    client = MagicMock()
+    client.get.side_effect = [
+        _redirect_response("https://site.com/final"),
+        _ok_response(),
+    ]
+    with patch.object(url_safety, "resolves_to_blocked_host", return_value=False):
+        resp = safe_get(client, "https://site.com/start")
+    assert resp.is_redirect is False
+    assert client.get.call_count == 2
+
+
+def test_guard_url_raises_on_blocked():
+    with patch.object(url_safety, "resolves_to_blocked_host", return_value=True):
+        with pytest.raises(BlockedHostError):
+            guard_url("http://127.0.0.1/x")
+
+
+@pytest.mark.parametrize("address,public", [
+    ("8.8.8.8", True),
+    ("2001:4860:4860::8888", True),
+    ("100.64.1.1", False),        # carrier-grade NAT
+    ("192.0.2.1", False),         # documentation range
+    ("::ffff:10.0.0.1", False),   # IPv4-mapped private
+    ("::ffff:8.8.8.8", True),     # IPv4-mapped public
+    ("224.0.0.1", False),         # multicast
+    ("fd00::1", False),           # unique local
+    ("64:ff9b::a9fe:a9fe", False),  # NAT64 to the metadata address
+    ("64:ff9b::808:808", True),     # NAT64 to a public address
+    ("::7f00:1", False),            # IPv4-compatible loopback
+    ("2002:a9fe:a9fe::", False),    # 6to4 to link-local
+    ("2002:808:808::", True),       # 6to4 to a public address
+    ("fec0::1", False),             # deprecated site-local
+    ("::1", False),
+])
+def test_is_public_ip(address, public):
+    assert is_public_ip(ipaddress.ip_address(address)) is public