test: 补充知识处理、检索与 Agent 回归用例

覆盖切块、向量检索、工具调用和历史预算；模型相关用例使用桩。

backend/tests/knowledge/test_enhanced_website_reader.py

@@ -0,0 +1,780 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import socket
+import unittest
+from unittest.mock import patch, MagicMock, Mock
+from bs4 import BeautifulSoup
+import httpx
+
+from app.knowledge.crawl_scope import DEFAULT_CRAWL_SCOPE, CrawlScope
+from app.knowledge.enhanced_website_reader import (
+    EnhancedWebsiteReader,
+    FAILURE_ADVICE,
+    FailureCause,
+)
+
+
+_RealClient = httpx.Client
+
+
+def serving(handler):
+    """The reader's HTTP client, answering every request with `handler(request)`:
+    real httpx responses, headers and streaming included."""
+    return patch(
+        'app.knowledge.enhanced_website_reader.httpx.Client',
+        side_effect=lambda **kwargs: _RealClient(transport=httpx.MockTransport(handler), **kwargs),
+    )
+
+
+class TestEnhancedWebsiteReader(unittest.TestCase):
+    """Test cases for EnhancedWebsiteReader"""
+
+    def setUp(self):
+        """Set up test environment"""
+        self.reader = EnhancedWebsiteReader(
+            max_depth=2,
+            max_links=5,
+            min_content_length=50
+        )
+
+        # The fetch path now runs an SSRF guard (url_safety.resolves_to_blocked_host)
+        # which does a DNS lookup — stub it to a fixed public IP so tests don't hit
+        # the network and aren't blocked.
+        dns_patcher = patch(
+            'app.knowledge.url_safety.socket.getaddrinfo',
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
+        )
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+        
+        # Create a simple HTML response for testing
+        self.test_html = """
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <title>Test Page</title>
+            <style>
+                .hidden { display: none; }
+            </style>
+            <script>console.log('This should be ignored');</script>
+        </head>
+        <body>
+            <header>
+                <nav>
+                    <a href="/page1">Page 1</a>
+                    <a href="/page2">Page 2</a>
+                </nav>
+            </header>
+            <div class="banner">Banner content to be ignored</div>
+            <main>
+                <h1>Main Content</h1>
+                <p>This is the main content of the page. It should be extracted properly.</p>
+                <p>Additional paragraph with meaningful content.</p>
+            </main>
+            <div class="sidebar">
+                <h2>Sidebar</h2>
+                <p>Sidebar content should be ignored.</p>
+            </div>
+            <div class="content">
+                <h2>Additional Content</h2>
+                <p>More content in a div with class 'content'.</p>
+            </div>
+            <div id="post-content">
+                <h2>Post Content</h2>
+                <p>Content in a div with id 'post-content'.</p>
+            </div>
+            <div>
+                <h2>Generic Content</h2>
+                <p>Content in a generic div without special class or id.</p>
+                <p>This is a good paragraph with substantial text that should be detected by density extraction.</p>
+                <p>Another good paragraph that helps identify this div as having high text density.</p>
+            </div>
+            <footer>
+                <p>Footer content to be ignored.</p>
+            </footer>
+            <div class="hidden">
+                <p>This should be ignored because it's hidden.</p>
+            </div>
+        </body>
+        </html>
+        """
+        self.soup = BeautifulSoup(self.test_html, 'html.parser')
+        
+    def test_extracts_every_content_block_on_the_page(self):
+        """All of a page's content blocks are kept, not just the first one.
+
+        The old walk returned the first container clearing min_content_length,
+        so a page like this one yielded only <main> and silently dropped the
+        three sibling blocks below it.
+        """
+        content = self.reader._extract_main_content(self.soup)
+        self.assertIn("This is the main content of the page", content)
+        self.assertIn("More content in a div with class 'content'", content)
+        self.assertIn("Content in a div with id 'post-content'", content)
+        self.assertIn("Content in a generic div", content)
+
+    def test_a_banner_before_the_content_does_not_win(self):
+        """A short block that merely clears the floor must not beat the page.
+
+        This is eazzyliving.co.uk: its first <article> was a 108-character
+        search widget, and returning it discarded ~4,500 characters of page.
+        The customer retried four times and left.
+        """
+        html = """
+        <html><body>
+            <article>Search properties. Search by city, university, or
+            neighbourhood to find verified student rooms and studios.</article>
+            <div class="listings">
+                <h1>Student accommodation in Leeds</h1>
+                <p>%s</p>
+            </div>
+        </body></html>
+        """ % ("Verified rooms close to campus with bills included. " * 40)
+
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Student accommodation in Leeds", content)
+        self.assertIn("Verified rooms close to campus", content)
+        self.assertGreater(len(content), 1000)
+
+    def test_the_page_title_survives_when_it_sits_in_a_page_header(self):
+        """Docs themes put the <h1> in a <header>, which selection treats as chrome.
+
+        docs.replypier.site lost "Quickstart: Launch Your AI Support Agent in
+        5 Minutes" that way. A page's title is its strongest retrieval signal,
+        so losing it makes the page harder for an agent to find, not just
+        shorter.
+        """
+        body = "Run the stack on your own infrastructure with the CLI. " * 20
+        html = f"""
+        <html><body>
+            <header><h1>Quickstart: Launch in 5 Minutes</h1></header>
+            <div class="prose"><p>{body}</p></div>
+            <footer>GitHub Discord</footer>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Quickstart: Launch in 5 Minutes", content)
+        self.assertIn("own infrastructure", content)
+        self.assertNotIn("Discord", content, "footer chrome should stay out")
+
+    def test_a_page_with_no_h1_falls_back_to_the_document_title(self):
+        """<title> lives in <head>, which _clean_soup strips.
+
+        Reading it after the clean would always find nothing, so a page with no
+        <h1> would silently have no title at all.
+        """
+        body = "Refund terms and conditions apply to all orders. " * 20
+        html = f"""
+        <html><head><title>Refund Policy</title></head>
+        <body><div class="c"><p>{body}</p></div></body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertIn("Refund Policy", content)
+
+    def test_the_title_is_not_duplicated_when_already_in_the_content(self):
+        """The usual case: the <h1> is inside the content container already."""
+        body = "Every plan we offer, billed monthly or yearly. " * 20
+        html = f"""
+        <html><body>
+            <main><h1>Plans and pricing</h1><p>{body}</p></main>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+
+        self.assertIn("Plans and pricing", content)
+        self.assertEqual(content.count("Plans and pricing"), 1,
+                         "the heading must not be prepended on top of itself")
+
+    def test_a_page_of_only_boilerplate_stays_below_the_floor(self):
+        """Nav chrome must not pass as content.
+
+        www.solcontrol.ca renders its catalogue with JavaScript; all the served
+        HTML holds is a phone number and a login link. That cleared the
+        100-character floor, so it was stored as the page's content *and* it
+        suppressed the browser fallback that reads the site properly. Staying
+        under the floor is what routes the page to Crawl4AI in _process_url.
+        """
+        html = """
+        <html><body>
+            <header><a href="/login">Welcome, Guest - Login</a>
+                    <a href="tel:905-230-8468">905-230-8468</a></header>
+            <nav><a href="/a">Products</a><a href="/b">About</a></nav>
+            <footer><a href="mailto:info@solcontrol.ca">info@solcontrol.ca</a></footer>
+        </body></html>
+        """
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertLess(len(content), self.reader.min_content_length)
+
+    def test_extract_content_by_density(self):
+        """Density extraction still runs when no container holds the page."""
+        html = """
+        <html><body>
+            <span>x</span>
+            <div><p>%s</p><p>%s</p></div>
+        </body></html>
+        """ % ("A good paragraph with substantial text. " * 5,
+               "Another good paragraph that carries the page. " * 5)
+        content = self.reader._extract_main_content(BeautifulSoup(html, 'html.parser'))
+        self.assertIn("A good paragraph with substantial text", content)
+
+    def test_clean_soup(self):
+        """Test cleaning of unwanted elements from HTML"""
+        # Create a copy for testing
+        soup_copy = BeautifulSoup(str(self.soup), 'html.parser')
+        self.reader._clean_soup(soup_copy)
+        
+        # Check that truly unwanted elements are removed (scripts, styles, hidden elements)
+        self.assertIsNone(soup_copy.find('script'))
+        self.assertIsNone(soup_copy.find('style'))
+        self.assertIsNone(soup_copy.find(class_='hidden'))
+        
+        # Check that navigation elements are removed (they contain menu links, not main content)
+        self.assertIsNone(soup_copy.find('nav'))
+        
+        # Check that sidebar elements are removed
+        self.assertIsNone(soup_copy.find(class_='sidebar'))
+        
+        # Check that main content elements are kept (header and footer may contain some content)
+        self.assertIsNotNone(soup_copy.find('header'))
+        self.assertIsNotNone(soup_copy.find('footer'))
+        self.assertIsNotNone(soup_copy.find('main'))
+        
+    def test_canonical_url(self):
+        """Fragments/trailing slashes are stripped so page variants collapse."""
+        c = self.reader._canonical_url
+        self.assertEqual(c('https://x.com/'), 'https://x.com')
+        self.assertEqual(c('https://x.com/#features'), 'https://x.com')
+        self.assertEqual(c('https://x.com/#pricing'), 'https://x.com')
+        self.assertEqual(c('https://x.com/pricing/'), 'https://x.com/pricing')
+        self.assertEqual(c('https://x.com/blogs#top'), 'https://x.com/blogs')
+        # The homepage and its anchored variants all canonicalize to one id.
+        variants = {c(u) for u in [
+            'https://x.com', 'https://x.com/', 'https://x.com/#a', 'https://x.com/#b'
+        ]}
+        self.assertEqual(len(variants), 1)
+
+    def test_looks_like_bot_challenge(self):
+        """Bot-check interstitials are detected so they aren't stored as content."""
+        f = self.reader._looks_like_bot_challenge
+        # The exact wp.com interstitial from the reported bug (strong marker).
+        self.assertTrue(f(
+            "Checking your browser This will only take a few seconds... "
+            "Secured by wp.com (URL: https://wordpress.com)"
+        ))
+        # Cloudflare markers.
+        self.assertTrue(f("cf-browser-verification"))
+        self.assertTrue(f("Just a moment..."))  # weak, but short
+        # Real content is not flagged, even if long and mentioning a weak phrase.
+        long_text = ("Our pricing is simple. " * 40) + "checking your browser settings is optional."
+        self.assertFalse(f(long_text))
+        self.assertFalse(f("Welcome to our pricing page. Plans start at $10 per seat."))
+        self.assertFalse(f(""))
+        self.assertFalse(f(None))
+
+    def test_extract_links_dedupes_page_variants(self):
+        """The homepage linked via #anchors and trailing slash yields one link."""
+        html = """
+        <html><body>
+          <a href="https://site.com/#features">Features</a>
+          <a href="https://site.com/#pricing">Pricing</a>
+          <a href="https://site.com/">Home</a>
+          <a href="https://site.com/docs">Docs</a>
+          <a href="https://site.com/docs/">Docs slash</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com')
+        # All homepage variants collapse (and equal the base, so are dropped);
+        # /docs and /docs/ collapse to a single canonical link.
+        self.assertEqual(links, ['https://site.com/docs'])
+
+    def test_extract_links_stays_on_the_seed_host_by_default(self):
+        """Default scope is the seed's own host: a help-center crawl must not
+        wander onto the marketing site, and a lookalike domain never matches."""
+        html = """
+        <html><body>
+          <a href="https://site.com/a">same</a>
+          <a href="https://www.site.com/b">www of the same host</a>
+          <a href="https://blog.site.com/c">subdomain (same registrable)</a>
+          <a href="https://evilsite.com/d">lookalike suffix</a>
+          <a href="https://other.com/e">unrelated</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com')
+        self.assertIn('https://site.com/a', links)
+        # In scope, and stored under the seed's spelling of the host.
+        self.assertIn('https://site.com/b', links)
+        self.assertNotIn('https://www.site.com/b', links)
+        self.assertNotIn('https://blog.site.com/c', links)
+        self.assertNotIn('https://evilsite.com/d', links)
+        self.assertNotIn('https://other.com/e', links)
+
+    def test_extract_links_domain_scope_spans_subdomains(self):
+        """The opt-in 'domain' scope keeps the previous reach across subdomains,
+        still by registrable-domain equality (never a suffix match)."""
+        reader = EnhancedWebsiteReader(crawl_scope='domain')
+        html = """
+        <html><body>
+          <a href="https://blog.site.com/b">subdomain</a>
+          <a href="https://evilsite.com/c">lookalike suffix</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = reader._extract_links(soup, 'https://site.com')
+        self.assertIn('https://blog.site.com/b', links)
+        self.assertNotIn('https://evilsite.com/c', links)
+
+    def test_extract_links_collapses_www_variants_onto_the_seed_host(self):
+        """A site that links both spellings stored the page twice — 4 of the 50
+        pages in a real paywithatoa.co.uk crawl were www/bare duplicates."""
+        html = """
+        <html><body>
+          <a href="https://www.site.com/terms">terms via www</a>
+          <a href="https://site.com/terms">terms bare</a>
+          <a href="https://www.site.com/privacy">privacy via www</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com/')
+        self.assertEqual(links, ['https://site.com/terms', 'https://site.com/privacy'])
+
+        # Seeded at the www spelling, the crawl keeps that one instead.
+        reader = EnhancedWebsiteReader()
+        links = reader._extract_links(soup, 'https://www.site.com/')
+        self.assertEqual(links, ['https://www.site.com/terms', 'https://www.site.com/privacy'])
+
+    def test_extract_links_skips_non_http_schemes(self):
+        """mailto:/tel: hrefs are contact actions, not pages. urljoin leaves them
+        untouched, and 'https://' + 'mailto:a@site.com' parses as host site.com
+        with 'mailto:a' as userinfo — so they used to be crawled as pages."""
+        html = """
+        <html><body>
+          <a href="mailto:hello@site.com">Email us</a>
+          <a href="tel:+441234567890">Call us</a>
+          <a href="javascript:void(0)">Menu</a>
+          <a href="/pricing">Pricing</a>
+        </body></html>
+        """
+        soup = BeautifulSoup(html, 'html.parser')
+        links = self.reader._extract_links(soup, 'https://site.com/help')
+        self.assertEqual(links, ['https://site.com/pricing'])
+
+    def test_normalize_url_leaves_non_http_schemes_alone(self):
+        """Prefixing 'mailto:a@site.com' with https:// produced a fetchable
+        'https://mailto:a@site.com' — the source of the mangled stored pages."""
+        self.assertEqual(self.reader._normalize_url('mailto:a@site.com'), 'mailto:a@site.com')
+        self.assertEqual(self.reader._normalize_url('site.com/x'), 'https://site.com/x')
+        self.assertEqual(self.reader._normalize_url('http://site.com'), 'http://site.com')
+
+    def test_get_primary_domain_strips_port_and_www(self):
+        self.assertEqual(self.reader._get_primary_domain('https://www.site.com:8443/x'), 'site.com')
+        self.assertEqual(self.reader._get_primary_domain('https://a.b.example.co.uk/x'), 'example.co.uk')
+
+    def test_crawl_with_successful_request(self):
+        """Test crawling with successful HTTP requests"""
+        requests = []
+
+        def site(request):
+            requests.append(request)
+            return httpx.Response(200, html=self.test_html)
+
+        with serving(site):
+            result = self.reader.crawl('https://example.com')
+
+        self.assertTrue(requests)
+        self.assertIn('https://example.com', result)
+        self.assertIn("Main Content", result['https://example.com'])
+
+    @patch('time.sleep', return_value=None)  # Skip actual sleeping
+    def test_crawl_with_retries(self, mock_sleep):
+        """Test crawling with retries on failed requests"""
+        attempts = []
+
+        def flaky_home(request):
+            if request.url.path != "/":
+                return httpx.Response(404)
+            attempts.append(request)
+            if len(attempts) == 1:
+                return httpx.Response(500)  # HTTP error
+            if len(attempts) == 2:
+                raise httpx.ReadTimeout("slow", request=request)  # transport error
+            return httpx.Response(200, html=self.test_html)
+
+        with serving(flaky_home):
+            result = self.reader.crawl('https://example.com')
+
+        self.assertEqual(len(attempts), 3)
+        self.assertIn('https://example.com', result)
+        self.assertIn("Main Content", result['https://example.com'])
+
+    def test_read_method(self):
+        """Test the read method to ensure it returns proper Document objects"""
+        with serving(lambda request: httpx.Response(200, html=self.test_html)):
+            documents = self.reader.read('https://example.com')
+        
+        # Verify documents are created correctly
+        self.assertTrue(len(documents) > 0)
+        self.assertIn("Main Content", documents[0].content)
+        
+        # Verify metadata format
+        self.assertEqual(documents[0].meta_data['url'], 'https://example.com')
+        self.assertEqual(documents[0].meta_data['chunk'], 1)
+        self.assertIsInstance(documents[0].meta_data['chunk_size'], int)
+        
+        # Verify ID and name
+        self.assertEqual(documents[0].id, 'https://example.com')
+        # Verify that name is the original source URL
+        self.assertEqual(documents[0].name, 'https://example.com')
+
+
+if __name__ == '__main__':
+    unittest.main()
+
+
+class TestUnreachableHostsFailFast(unittest.TestCase):
+    """A host we never reached fails the same way every attempt.
+
+    The old ladder — three tries with exponential backoff, then a headless
+    browser — spent 130 seconds on mateusoliveira.com (which drops the SYN on
+    80 and 443) and ~10 seconds on each domain with no A record, holding one of
+    only two queue slots for the whole time.
+    """
+
+    def setUp(self):
+        self.reader = EnhancedWebsiteReader(max_depth=1, max_links=1, min_content_length=50)
+        dns_patcher = patch(
+            'app.knowledge.url_safety.socket.getaddrinfo',
+            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))],
+        )
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def _crawl_with(self, error):
+        """Run one page through the fetch path with `error` raised every time."""
+        scope = CrawlScope.for_seed("https://example.com", DEFAULT_CRAWL_SCOPE)
+        with patch('app.knowledge.enhanced_website_reader.httpx.Client') as client, \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback') as browser, \
+             patch('app.knowledge.enhanced_website_reader.time.sleep') as sleep:
+            client.return_value.__enter__.return_value.stream.side_effect = error
+            browser.return_value.is_available = True
+            browser.return_value.fetch_with_browser.return_value = (None, None, None)
+            result = self.reader._process_url(("https://example.com", 1), scope)
+        return result, client.return_value.__enter__.return_value.stream, browser, sleep
+
+
+    def test_a_domain_that_does_not_resolve_is_tried_once(self):
+        error = httpx.ConnectError("[Errno 8] nodename nor servname provided, or not known")
+        result, get, browser, _ = self._crawl_with(error)
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1, "no point retrying a name that does not resolve")
+        browser.return_value.fetch_with_browser.assert_not_called()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.DNS: 1})
+
+    def test_a_dropped_connection_attempt_is_tried_once(self):
+        result, get, browser, _ = self._crawl_with(httpx.ConnectTimeout("timed out"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1)
+        browser.return_value.fetch_with_browser.assert_not_called()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.TIMEOUT: 1})
+
+    def test_a_refused_connection_still_gets_the_browser(self):
+        """Could be a TLS quirk rather than a dead host, so give Chromium a turn."""
+        result, get, browser, _ = self._crawl_with(httpx.ConnectError("Connection refused"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, 1, "a refusal will not change in two seconds")
+        browser.return_value.fetch_with_browser.assert_called_once()
+        self.assertEqual(self.reader._failure_causes, {FailureCause.CONNECTION: 1})
+
+    def test_a_slow_page_is_still_retried(self):
+        """Read timeouts are the transient case retries exist for."""
+        result, get, browser, _ = self._crawl_with(httpx.ReadTimeout("too slow"))
+
+        self.assertIsNone(result)
+        self.assertEqual(get.call_count, self.reader.max_retries)
+        browser.return_value.fetch_with_browser.assert_called_once()
+
+    def test_our_own_bugs_are_not_blamed_on_the_customers_site(self):
+        """A crash in our parsing must not be reported as the site being down.
+
+        Everything inside the fetch block — BeautifulSoup, link extraction,
+        content selection — can raise. Classifying those as a connection
+        failure told the customer "your site refused the connection" when what
+        actually happened was our own AttributeError.
+        """
+        reader = EnhancedWebsiteReader()
+        for error in (AttributeError("'NoneType' has no attribute 'find'"),
+                      KeyError("href"),
+                      ValueError("bad parse")):
+            cause, _, _ = reader._classify_transport_error(error)
+            self.assertEqual(cause, FailureCause.UNKNOWN)
+            self.assertNotIn(cause, FAILURE_ADVICE,
+                             "UNKNOWN must fall back to the generic message")
+
+    def test_our_own_pool_exhaustion_is_not_blamed_on_the_site(self):
+        """PoolTimeout means we never sent the request, so the site is not slow."""
+        reader = EnhancedWebsiteReader()
+        cause, retry, _ = reader._classify_transport_error(httpx.PoolTimeout("pool full"))
+        self.assertEqual(cause, FailureCause.UNKNOWN)
+        self.assertTrue(retry, "a saturated pool does free up")
+
+    def test_name_resolution_is_detected_from_the_socket_error(self):
+        """Not from the message text, which other errors can imitate."""
+        reader = EnhancedWebsiteReader()
+
+        gai = httpx.ConnectError("connection failed")
+        gai.__cause__ = socket.gaierror(8, "nodename nor servname provided")
+        self.assertEqual(reader._classify_transport_error(gai)[0], FailureCause.DNS)
+
+        # A refusal that merely mentions a name must not be read as DNS, or we
+        # would skip the browser fallback for a host that is actually up.
+        refused = httpx.ConnectError("Connection refused by name-based vhost")
+        cause, _, browser = reader._classify_transport_error(refused)
+        self.assertEqual(cause, FailureCause.CONNECTION)
+        self.assertTrue(browser)
+
+
+class TestWwwApexFallback(unittest.TestCase):
+    """A ``www`` host with no address falls back to the apex.
+
+    Production job 392: a customer pasted https://www.utsabzone.store, whose
+    ``www`` CNAME is broken while the apex serves the site fine. They were told
+    "the domain does not resolve" — accurate about the name they gave us, and
+    useless, because the site was up.
+    """
+
+    def setUp(self):
+        self.reader = EnhancedWebsiteReader(max_depth=1, max_links=1)
+
+    def _resolver(self, resolvable):
+        """getaddrinfo that only knows about `resolvable` hosts."""
+        def fake(host, *args, **kwargs):
+            if host in resolvable:
+                return [(2, 1, 6, "", ("93.184.216.34", 0))]
+            raise socket.gaierror(8, "nodename nor servname provided, or not known")
+        return fake
+
+    def test_a_dead_www_seed_falls_back_to_the_apex(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"utsabzone.store"})):
+            out = self.reader._seed_with_resolvable_host("https://www.utsabzone.store")
+        self.assertEqual(out, "https://utsabzone.store")
+
+    def test_a_working_www_seed_is_left_alone(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"www.example.com", "example.com"})):
+            out = self.reader._seed_with_resolvable_host("https://www.example.com/pricing")
+        self.assertEqual(out, "https://www.example.com/pricing")
+
+    def test_a_non_www_seed_costs_no_lookup(self):
+        """The common case must not pay for DNS it does not need."""
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo') as gai:
+            out = self.reader._seed_with_resolvable_host("https://example.com")
+        self.assertEqual(out, "https://example.com")
+        gai.assert_not_called()
+
+    def test_both_hosts_dead_keeps_the_url_the_user_gave(self):
+        """So the error still names what they typed."""
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver(set())):
+            out = self.reader._seed_with_resolvable_host("https://www.jourico.com")
+        self.assertEqual(out, "https://www.jourico.com")
+
+    def test_the_path_query_and_port_survive_the_swap(self):
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"shop.example"})):
+            out = self.reader._seed_with_resolvable_host(
+                "https://www.shop.example:8443/catalogue?page=2")
+        self.assertEqual(out, "https://shop.example:8443/catalogue?page=2")
+
+    def test_credentials_in_the_url_are_never_rewritten(self):
+        url = "https://user:pass@www.example.com/"
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=self._resolver({"example.com"})):
+            self.assertEqual(self.reader._seed_with_resolvable_host(url), url)
+
+    def test_an_unverifiable_apex_is_never_swapped_in(self):
+        """A lookup that errors oddly must not count as "the apex is fine".
+
+        Otherwise we would rewrite the seed to a host we never confirmed and
+        report the failure against the wrong name.
+        """
+        def odd(host, *args, **kwargs):
+            if host.startswith("www."):
+                raise socket.gaierror(8, "not known")
+            raise UnicodeError("malformed host")
+
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo', side_effect=odd):
+            out = self.reader._seed_with_resolvable_host("https://www.example.com")
+        self.assertEqual(out, "https://www.example.com")
+
+    def test_the_swapped_apex_is_still_checked_by_the_ssrf_guard(self):
+        """The fallback derives a host the user never typed, so prove it is guarded.
+
+        A dead ``www`` whose apex points at loopback is the vector: without the
+        per-hop check in safe_get, the rewrite would reach an internal address.
+        """
+        from app.knowledge.url_safety import BlockedHostError, guard_url
+
+        def internal_apex(host, *args, **kwargs):
+            if host == "attacker.test":
+                return [(2, 1, 6, "", ("127.0.0.1", 0))]
+            raise socket.gaierror(8, "nodename nor servname provided")
+
+        with patch('app.knowledge.enhanced_website_reader.socket.getaddrinfo',
+                   side_effect=internal_apex):
+            swapped = self.reader._seed_with_resolvable_host("https://www.attacker.test")
+        self.assertEqual(swapped, "https://attacker.test")
+
+        with patch('app.knowledge.url_safety.socket.getaddrinfo', side_effect=internal_apex):
+            with self.assertRaises(BlockedHostError):
+                guard_url(swapped)
+
+    def test_the_swapped_apex_stays_inside_the_crawl_scope(self):
+        """Otherwise the fallback would fetch one page and refuse every link."""
+        scope = CrawlScope.for_seed("https://www.utsabzone.store", DEFAULT_CRAWL_SCOPE)
+        self.assertTrue(scope.allows("https://utsabzone.store/products"))
+
+
+INSTALLERS = [
+    "/downloads/linux/amd64/installer/ian-vpn.tar.gz",
+    "/downloads/linux/amd64/installer/ian-vpn.deb",
+    "/downloads/linux/amd64/installer/ian-vpn.rpm",
+]
+TEXT = " ".join(["Our VPN keeps your connection private on every network."] * 5)
+
+
+def vpn_site(request):
+    """A home page that links to a FAQ and to its installers, as ian-vpn.site does."""
+    path = request.url.path
+    if path == "/":
+        links = "".join(f'<a href="{href}">Download</a>' for href in ["/faqs", *INSTALLERS])
+        return httpx.Response(200, html=f"<main><h1>IAN VPN</h1><p>{TEXT}</p>{links}</main>")
+    if path == "/faqs":
+        return httpx.Response(200, html=f"<main><h1>FAQs</h1><p>{TEXT}</p></main>")
+    if path in INSTALLERS:
+        return httpx.Response(200, headers={"content-type": "application/octet-stream"},
+                              content=b"\x7fELF\x00\x00" * 100_000)
+    return httpx.Response(404)
+
+
+class TestDownloadsAreNotPages(unittest.TestCase):
+    """Seen in production: ten links on ian-vpn.site became 149,500 chunks,
+    almost all of them its installers read as text."""
+
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def test_the_pages_are_read_and_the_installers_skipped(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback') as browser:
+            result = reader.crawl("https://ian-vpn.site")
+
+        self.assertEqual(sorted(result), ["https://ian-vpn.site", "https://ian-vpn.site/faqs"])
+        browser.return_value.fetch_with_browser.assert_not_called()
+        # Skipped, not failed: they say nothing about whether the site works
+        self.assertEqual(reader._failure_causes, {})
+        self.assertEqual(reader._failed_crawls, 0)
+
+    def test_the_documents_come_from_the_pages_only(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            documents = reader.read("https://ian-vpn.site")
+
+        self.assertEqual({d.meta_data['url'] for d in documents},
+                         {"https://ian-vpn.site", "https://ian-vpn.site/faqs"})
+        self.assertLess(len(documents), 10)
+
+
+class FakeDeadline:
+    expired = False
+
+
+class TestTheCrawlStopsAtItsDeadline(unittest.TestCase):
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def test_no_new_page_is_read_once_time_is_up(self):
+        deadline = FakeDeadline()
+        requested = []
+
+        def site(request):
+            requested.append(request.url.path)
+            deadline.expired = True  # the home page used the last of the time
+            return vpn_site(request)
+
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50, deadline=deadline)
+        with serving(site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            result = reader.crawl("https://ian-vpn.site")
+
+        self.assertEqual(requested, ["/"])
+        self.assertEqual(list(result), ["https://ian-vpn.site"])
+
+    def test_without_a_deadline_the_crawl_runs_to_its_link_limit(self):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            result = reader.crawl("https://ian-vpn.site")
+        self.assertEqual(len(result), 2)
+
+
+class TestASeedThatIsNotAPage(unittest.TestCase):
+    """A file given as a website must say so, not blame JavaScript."""
+
+    def setUp(self):
+        dns_patcher = patch('app.knowledge.url_safety.socket.getaddrinfo',
+                            return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
+        self.addCleanup(dns_patcher.stop)
+        dns_patcher.start()
+
+    def crawl(self, response):
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(lambda request: response), \
+             patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            return reader, reader.crawl("https://example.com/brochure.pdf")
+
+    def test_a_file_given_as_the_website_is_named_as_one(self):
+        reader, result = self.crawl(httpx.Response(200, headers={"content-type": "application/pdf"},
+                                                   content=b"%PDF-1.4"))
+        self.assertEqual(result, {})
+        self.assertEqual(reader.dominant_failure_cause(), FailureCause.NOT_A_PAGE)
+        self.assertTrue(FAILURE_ADVICE[FailureCause.NOT_A_PAGE].startswith("{url} is a file"))
+
+    def test_an_oversized_page_is_not_called_a_file(self):
+        reader, _ = self.crawl(httpx.Response(200, headers={"content-type": "text/html",
+                                                            "content-length": str(50 * 1024 * 1024)},
+                                              content=b""))
+        self.assertIsNone(reader.dominant_failure_cause())
+
+    def test_linked_downloads_are_never_a_cause(self):
+        """Only the address given: a site's installer links say nothing about it."""
+        reader = EnhancedWebsiteReader(max_depth=2, max_links=10, min_content_length=50)
+        with serving(vpn_site), patch('app.knowledge.enhanced_website_reader.get_crawl4ai_fallback'):
+            reader.crawl("https://ian-vpn.site")
+        self.assertEqual(reader._failure_causes, {})