feat(knowledge): 任务队列、独立 worker 与 pgvector 向量索引

知识处理改为数据库任务队列，独立 worker 做切块与向量化，失败可重试。

backend/app/knowledge/sitemap_reader.py

@@ -0,0 +1,85 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from typing import Callable, Dict, Optional
+
+from app.core.logger import get_logger
+from app.knowledge.crawl_scope import SCOPE_DOMAIN, CrawlScope
+from app.knowledge.enhanced_website_reader import EnhancedWebsiteReader
+from app.knowledge.sitemap_parser import fetch_sitemap_urls
+
+logger = get_logger(__name__)
+
+
+class SitemapReader(EnhancedWebsiteReader):
+    """Crawl exactly the pages listed in a ``sitemap.xml`` — no ``<a href>`` following.
+
+    Reuses EnhancedWebsiteReader's whole per-page pipeline (httpx fetch, Crawl4AI
+    fallback, bot-challenge detection, content extraction, Document creation, and
+    the ``name = <sitemap url>`` / ``id = <page url>`` grouping); only the URL
+    source differs — pages come from the sitemap rather than BFS link discovery.
+
+    Construct with ``max_depth=1`` so ``_process_url`` never yields child links,
+    and ``max_links`` = the plan's ``max_sub_pages`` so the discovered page list
+    is capped at ingest.
+    """
+
+    def crawl(
+        self,
+        url: str,
+        starting_depth: int = 1,
+        on_document_callback: Optional[Callable[[str, str], None]] = None,
+        on_url_crawled_callback: Optional[Callable[[str], None]] = None,
+    ) -> Dict[str, str]:
+        url = self._normalize_url(url)
+        # Same www/apex fallback as a website crawl - a sitemap URL on a broken
+        # www host is the identical customer problem.
+        url = self._seed_with_resolvable_host(url)
+
+        # Reset per-crawl state exactly as the base crawl does — the counters are
+        # read by the progress logs and by _raise_if_nothing_stored (a fully
+        # bot-blocked sitemap must still fail with the "add manually" message).
+        self._visited = set()
+        self._urls_to_crawl = []
+        self._crawled_pages_count = 0
+        self._successful_crawls = 0
+        self._failed_crawls = 0
+        self._challenge_blocked = 0
+        self._failure_causes = {}
+
+        # A sitemap index legitimately lists pages on sibling hosts
+        # (help.example.com from example.com/sitemap.xml), so the pages it names
+        # are taken at registrable-domain scope — matching the parser's own
+        # cross-domain filter. Only <a href> following is host-scoped.
+        scope = CrawlScope.for_seed(url, SCOPE_DOMAIN)
+        page_urls = fetch_sitemap_urls(
+            url,
+            headers=self.headers,
+            timeout=self.timeout,
+            verify_ssl=self.verify_ssl,
+            max_urls=self.max_links,
+        )
+        # Exposed so the dispatcher can fail a sitemap that lists no pages
+        # (wrong URL / not a sitemap) instead of silently creating an empty source.
+        self._sitemap_page_count = len(page_urls)
+        logger.info(f"Sitemap {url}: crawling {len(page_urls)} listed page(s)")
+        if not page_urls:
+            return {}
+
+        seeds = [(page_url, starting_depth) for page_url in page_urls]
+        return self._run_crawl_queue(
+            seeds, scope, on_document_callback, on_url_crawled_callback
+        )