feat(knowledge): 任务队列、独立 worker 与 pgvector 向量索引

知识处理改为数据库任务队列，独立 worker 做切块与向量化，失败可重试。
2026-09-30 22:05:00 +08:00
1 parent 7937de5d7d
commit c7011040c3
@@ -0,0 +1,185 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+"""Which URLs a website crawl is allowed to follow.
+
+Two independent problems this decides, both seen on a real help-center import
+that stored 247 pages of which only ~102 were help articles:
+
+* **Scheme.** ``urljoin`` leaves a ``mailto:``/``tel:`` href untouched, and
+  prefixing a scheme-less value with ``https://`` turns ``mailto:a@example.com``
+  into ``https://mailto:a@example.com`` — which parses as host ``example.com``
+  with ``mailto:a`` as userinfo, so it passed the same-domain check and was
+  crawled as a page (then spawned ``https://mailto:a@example.com/pricing`` and
+  friends from the fetched body). Only http(s) URLs are crawlable.
+
+* **Reach.** Bounding a crawl by the *registrable domain* means a crawl seeded at
+  ``help.example.com`` also walks ``www.example.com``, ``example.com/blog/*`` and
+  every other host under ``example.com`` — the whole company web presence lands
+  in one knowledge source. The default is now the seed's own host; ``domain``
+  (previous behaviour) and ``path`` (seed host + seed path prefix) stay available
+  per source.
+"""
+
+from dataclasses import dataclass
+from typing import Optional
+from urllib.parse import ParseResult, urlparse, urlunparse
+
+from app.knowledge.domains import registrable_domain
+
+# Scope modes, narrowest first. Kept as plain strings: they travel through the
+# API request and the queue item's JSON metadata.
+SCOPE_PATH = "path"
+SCOPE_HOST = "host"
+SCOPE_DOMAIN = "domain"
+CRAWL_SCOPES = (SCOPE_PATH, SCOPE_HOST, SCOPE_DOMAIN)
+
+DEFAULT_CRAWL_SCOPE = SCOPE_HOST
+
+# Anything else (mailto:, tel:, sms:, javascript:, data:, ftp:) is a contact or
+# script action, not a page to fetch.
+CRAWLABLE_SCHEMES = ("http", "https")
+
+
+def url_scheme(url: str) -> str:
+    """Lowercased scheme of a URL, or ``''`` when it has none.
+
+    Not ``urlparse(url).scheme``: that reads the scheme-less ``example.com:8443/x``
+    as scheme ``example.com``. A real scheme has no ``.`` or ``/`` before the colon.
+    """
+    head, sep, _ = (url or "").strip().partition(":")
+    if not sep or not head or "." in head or "/" in head:
+        return ""
+    return head.lower()
+
+
+def is_crawlable_url(url: str) -> bool:
+    """True for an http(s) URL or a scheme-less one (a relative href still to be
+    resolved against its base). False for ``mailto:``/``tel:``/``javascript:``/…"""
+    scheme = url_scheme(url)
+    return scheme in CRAWLABLE_SCHEMES or not scheme
+
+
+def _parsed(url: str) -> Optional[ParseResult]:
+    """Parsed URL, or None when it can't be parsed — a malformed IPv6 literal
+    makes ``.hostname`` raise, and callers want "out of scope", not a traceback."""
+    try:
+        parsed = urlparse(url or "")
+        parsed.hostname  # raises here rather than at each use site
+    except ValueError:
+        return None
+    return parsed
+
+
+def _hostname(url: str) -> str:
+    """Host of a URL, or ``''`` if it has none / can't be parsed."""
+    parsed = _parsed(url)
+    return (parsed.hostname or "") if parsed else ""
+
+
+def _host_key(host: str) -> str:
+    """Host identity for comparisons: lowercased, ``www.`` dropped, so a site
+    linking between ``example.com`` and ``www.example.com`` stays in scope."""
+    host = (host or "").lower().strip().rstrip(".")
+    return host[4:] if host.startswith("www.") else host
+
+
+@dataclass(frozen=True)
+class CrawlScope:
+    """The boundary of one crawl, derived from its seed URL."""
+
+    mode: str
+    domain: str
+    host: str
+    seed_host: str
+    path_prefix: str
+
+    @classmethod
+    def for_seed(cls, seed_url: str, mode: str = DEFAULT_CRAWL_SCOPE) -> "CrawlScope":
+        """Scope for a crawl seeded at ``seed_url``. An unknown mode falls back to
+        the default rather than failing a queued crawl."""
+        if mode not in CRAWL_SCOPES:
+            mode = DEFAULT_CRAWL_SCOPE
+        parsed = _parsed(seed_url)
+        host = ((parsed.hostname or "") if parsed else "").lower()
+        return cls(
+            mode=mode,
+            domain=registrable_domain(host),
+            host=_host_key(host),
+            seed_host=host,
+            path_prefix=((parsed.path or "") if parsed else "").rstrip("/"),
+        )
+
+    def canonical(self, url: str) -> str:
+        """Put an in-scope URL on the seed's own spelling of the host.
+
+        ``example.com/terms`` and ``www.example.com/terms`` are one page, but
+        they are two page ids — so a site that links both ways stores the page
+        twice, burning two of the source's sub-page slots and feeding retrieval
+        and FAQ generation the same text twice.
+
+        Rewrites toward the SEED's host rather than always stripping ``www.``:
+        plenty of sites serve only one of the two, and the seed is the form the
+        user gave us and the crawl already fetched. Only the seed host's own
+        variant is touched — a genuine subdomain under ``domain`` scope is left
+        as it is.
+        """
+        parsed = _parsed(url)
+        if parsed is None or not parsed.hostname:
+            return url
+        host = parsed.hostname.lower()
+        if host == self.seed_host or _host_key(host) != self.host:
+            return url
+        netloc = f"{self.seed_host}:{parsed.port}" if parsed.port else self.seed_host
+        return urlunparse(parsed._replace(netloc=netloc))
+
+    def allows(self, url: str) -> bool:
+        """True if ``url`` may be fetched and stored as part of this crawl."""
+        if url_scheme(url) not in CRAWLABLE_SCHEMES:
+            return False
+
+        parsed = _parsed(url)
+        if parsed is None:
+            return False
+        # No userinfo: pages are never fetched with credentials, and this is what
+        # a leftover 'https://mailto:a@example.com/x' from an older crawl looks
+        # like ('mailto:a' as user:password) once the scheme has been mangled off.
+        if parsed.username or parsed.password:
+            return False
+
+        host = parsed.hostname or ""
+        if self.mode == SCOPE_DOMAIN:
+            # Equality, not a suffix match: 'evilexample.com'.endswith('example.com')
+            # would let one plan-limited source fan out across unrelated domains.
+            return bool(self.domain) and registrable_domain(host) == self.domain
+
+        if not self.host or _host_key(host) != self.host:
+            return False
+        if self.mode == SCOPE_HOST:
+            return True
+
+        # Path scope: compare on a segment boundary so a '/hc/en' seed keeps
+        # '/hc/en/articles/1' but not '/hc/entertainment'.
+        path = (parsed.path or "").rstrip("/")
+        return path == self.path_prefix or path.startswith(f"{self.path_prefix}/")
+
+    def describe(self) -> str:
+        """Short human-readable scope, for crawl logs."""
+        if self.mode == SCOPE_DOMAIN:
+            return f"domain {self.domain}"
+        if self.mode == SCOPE_HOST:
+            return f"host {self.host}"
+        return f"host {self.host} under {self.path_prefix or '/'}"