feat(modules): 帮助中心、工单、工作流与多渠道接入

补齐 FAQ、帮助中心、工单、可视化工作流及 Telegram/WhatsApp/Slack 等渠道模块。

backend/app/services/faq_generation.py

@@ -0,0 +1,431 @@
+"""
+Copyright 2024-2026 ReplyPier
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+FAQ generation pipeline: read the org's stored knowledge text (no re-crawl),
+extract grounded FAQs with the org's configured model, dedup against existing
+questions and insert drafts for review. Additive only — regeneration never
+edits or deletes existing FAQs. The drafting loop and dedup/category helpers
+here are shared with the import service (faq_import.py).
+"""
+
+import re
+from collections import deque
+from difflib import SequenceMatcher
+from typing import Awaitable, Callable, List, Optional, Tuple, TypeVar
+
+from fastapi import HTTPException
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from app.agents.faq_generator import FAQGeneratorAgent, GeneratedFAQ
+from app.core.config import settings
+from app.core.logger import get_logger
+from app.core.security import decrypt_api_key
+from app.knowledge.page_editor import NATURAL_ID_ORDER, PAGE_ID_EXPR
+from app.models.faq import FAQ, DEFAULT_FAQ_CATEGORY, FAQStatus
+from app.models.faq_generation_job import FAQGenerationJob, FAQJobStage, FAQJobType
+from app.models.knowledge import Knowledge
+from app.repositories.ai_config import AIConfigRepository
+from app.repositories.faq import FAQRepository
+from app.repositories.faq_generation_job import FAQGenerationJobRepository
+
+logger = get_logger(__name__)
+
+# Dedup lists injected into prompts are a small recent window for intra-run
+# steering only — the in-Python DedupState below (exact + fuzzy) is the real
+# guard covering every existing question, so the prompt stays cheap.
+MAX_PROMPT_QUESTIONS = 30
+
+# Fuzzy-dedup tuning: token-set Jaccard on content words; borderline scores are
+# confirmed with difflib on the normalized strings. Very short questions skip
+# fuzzy matching entirely (too collision-prone).
+FUZZY_JACCARD_DUP = 0.8
+FUZZY_JACCARD_MAYBE = 0.65
+FUZZY_SEQUENCE_DUP = 0.87
+FUZZY_MIN_CONTENT_TOKENS = 3
+
+# Tiny inline stopword set — enough to keep filler words from inflating
+# question similarity, without an NLP dependency.
+_STOPWORDS = frozenset(
+    "a an and are can could do does for from how i in is it my of on or the "
+    "there to we what when where which who why will with you your".split()
+)
+
+
+_PUNCT_RE = re.compile(r"[^\w\s]", re.UNICODE)
+
+BatchT = TypeVar("BatchT")
+
+
+def normalize_question(question: str) -> str:
+    """Casefold + strip punctuation/whitespace, so trivial rephrasings of the
+    same question dedup against each other."""
+    return " ".join(_PUNCT_RE.sub(" ", question.casefold()).split())
+
+
+class NoAIConfigError(Exception):
+    """The org has no active AI configuration to generate with."""
+
+
+def build_generator(db: Session, organization_id, job: Optional[FAQGenerationJob] = None) -> FAQGeneratorAgent:
+    """The org's configured extraction agent. When a job is given, every actual
+    provider call (retries and the plain-JSON fallback's second call included)
+    increments the job's llm_calls — the metering source of truth."""
+    config = AIConfigRepository(db).get_active_config(organization_id)
+    if not config:
+        raise NoAIConfigError(
+            "No active AI configuration. Set up an AI model in AI Configuration first."
+        )
+    generator = FAQGeneratorAgent(
+        api_key=decrypt_api_key(config.encrypted_api_key),
+        model_name=config.model_name,
+        model_type=config.model_type.value if hasattr(config.model_type, "value") else str(config.model_type),
+        base_url=(config.settings or {}).get('base_url'),
+    )
+    if job is not None:
+        job_repo = FAQGenerationJobRepository(db)
+        generator.on_llm_call = lambda: job_repo.increment_llm_calls(job.id)
+    return generator
+
+
+def load_source_pages(db: Session, knowledge: Knowledge, max_chars: Optional[int] = None) -> List[str]:
+    """One string of aggregated text per crawled page/sub-page of a source,
+    read straight from the org's vector table (content is stored alongside the
+    embeddings — no re-crawl needed). Pages kept in crawl order when the cap
+    truncates; chunk text within a page is naturally ordered. max_chars caps
+    each page and defaults to the model-agnostic floor."""
+    if not knowledge.schema or not knowledge.table_name:
+        # Source rows without a vector table (failed/partial ingestion).
+        return []
+    # Chunks written by app.knowledge.chunking (`::n` ids) are verbatim slices
+    # and join back with nothing between them; legacy agno `_n` chunks were
+    # cut on whitespace they dropped, so those keep a newline.
+    query = text(
+        f"SELECT {PAGE_ID_EXPR} AS page_id, "
+        "string_agg(content, CASE WHEN id ~ '::[0-9]+$' THEN '' ELSE E'\n' END "
+        f"ORDER BY {NATURAL_ID_ORDER}) AS body "
+        f'FROM {knowledge.schema}."{knowledge.table_name}" '
+        "WHERE name = :source GROUP BY 1 ORDER BY min(created_at) LIMIT :max_pages"
+    )
+    rows = db.execute(
+        query, {"source": knowledge.source, "max_pages": settings.FAQ_MAX_PAGES_PER_SOURCE}
+    ).fetchall()
+    max_chars = max_chars or settings.FAQ_MAX_BATCH_CHARS
+    return [row.body[:max_chars] for row in rows if row.body and row.body.strip()]
+
+
+def pack_batches(items: List[str], max_chars: Optional[int] = None, sep: str = "\n\n---\n\n") -> List[str]:
+    """Greedy-pack whole items (pages, lines) into batches of ~max_chars.
+    Oversized single items are truncated to max_chars, never split across
+    batches. Shared by generation (page items) and import (line items)."""
+    max_chars = max_chars or settings.FAQ_MAX_BATCH_CHARS
+    batches: List[str] = []
+    current: List[str] = []
+    size = 0
+    for item in items:
+        item = item[:max_chars]
+        if current and size + len(item) + len(sep) > max_chars:
+            batches.append(sep.join(current))
+            current, size = [], 0
+        current.append(item)
+        size += len(item) + len(sep)
+    if current:
+        batches.append(sep.join(current))
+    return batches
+
+
+def _content_tokens(normalized: str) -> frozenset:
+    return frozenset(t for t in normalized.split() if t not in _STOPWORDS)
+
+
+class DedupState:
+    """Tracks every question seen (existing + accepted this run).
+
+    Two layers: an exact normalized-set match, plus fuzzy similarity (token-set
+    Jaccard via an inverted index, difflib confirmation on borderline scores)
+    so rephrasings are caught without shipping the whole question list to the
+    LLM. A bounded recent-questions window is still exposed for prompts."""
+
+    def __init__(self, existing_questions: List[str]):
+        self.seen: set = set()
+        self._token_index: dict = {}  # content token -> set of seen keys
+        self._key_tokens: dict = {}  # seen key -> its content-token frozenset
+        self._recent = deque(existing_questions, maxlen=MAX_PROMPT_QUESTIONS)
+        for q in existing_questions:
+            self._remember(normalize_question(q))
+
+    def _remember(self, key: str) -> None:
+        if not key or key in self.seen:
+            return
+        self.seen.add(key)
+        tokens = _content_tokens(key)
+        self._key_tokens[key] = tokens
+        for token in tokens:
+            self._token_index.setdefault(token, set()).add(key)
+
+    def is_similar(self, key: str) -> bool:
+        """True when key fuzzily matches a seen question. Exact hits are the
+        caller's job (cheap set test); this covers rephrasings."""
+        tokens = _content_tokens(key)
+        if len(tokens) < FUZZY_MIN_CONTENT_TOKENS:
+            return False
+        candidates: set = set()
+        for token in tokens:
+            candidates |= self._token_index.get(token, set())
+        for candidate in candidates:
+            other = self._key_tokens[candidate]
+            union = len(tokens | other)
+            if not union:
+                continue
+            jaccard = len(tokens & other) / union
+            if jaccard >= FUZZY_JACCARD_DUP:
+                return True
+            if jaccard >= FUZZY_JACCARD_MAYBE and (
+                SequenceMatcher(None, key, candidate).ratio() >= FUZZY_SEQUENCE_DUP
+            ):
+                return True
+        return False
+
+    def accept_new(self, faqs: List[GeneratedFAQ]) -> List[GeneratedFAQ]:
+        accepted = []
+        for faq in faqs:
+            key = normalize_question(faq.question)
+            if not key or key in self.seen or self.is_similar(key):
+                continue
+            self._remember(key)
+            self._recent.append(faq.question)
+            accepted.append(faq)
+        return accepted
+
+    @property
+    def for_prompt(self) -> List[str]:
+        return list(self._recent)
+
+
+class CategoryMerger:
+    """Case-insensitive merge of model-emitted categories into the org's
+    existing spellings."""
+
+    def __init__(self, db: Session, organization_id):
+        self._known = {c.casefold(): c for c in FAQRepository(db).get_categories(organization_id)}
+
+    def merge(self, category: str) -> str:
+        cleaned = (category or "").strip() or DEFAULT_FAQ_CATEGORY
+        return self._known.setdefault(cleaned.casefold(), cleaned)
+
+    @property
+    def names(self) -> List[str]:
+        return list(self._known.values())
+
+
+async def draft_batches(
+    db: Session,
+    job: FAQGenerationJob,
+    batches: List[BatchT],
+    extract: Callable[[BatchT, DedupState], Awaitable[List[GeneratedFAQ]]],
+    retries_per_batch: int = 1,
+) -> List[Tuple[BatchT, GeneratedFAQ]]:
+    """Shared DRAFTING stage: run `extract` per batch with retry, dedup as we
+    go, report progress 30→90%. Raises only when every batch failed."""
+    job_repo = FAQGenerationJobRepository(db)
+    job_repo.update_progress(job.id, stage=FAQJobStage.DRAFTING, progress_percentage=30.0)
+    dedup = DedupState(FAQRepository(db).get_existing_questions(job.organization_id))
+
+    accepted: List[Tuple[BatchT, GeneratedFAQ]] = []
+    failures = 0
+    last_error: Optional[Exception] = None
+    for index, batch in enumerate(batches):
+        faqs = None
+        for attempt in range(1 + retries_per_batch):
+            try:
+                faqs = await extract(batch, dedup)
+                break
+            except Exception as e:
+                last_error = e
+                logger.warning(f"FAQ batch {index + 1}/{len(batches)} attempt {attempt + 1} failed: {e}")
+        if faqs is None:
+            failures += 1
+        else:
+            accepted.extend((batch, faq) for faq in dedup.accept_new(faqs))
+        job_repo.update_progress(
+            job.id, progress_percentage=30.0 + 60.0 * (index + 1) / len(batches)
+        )
+    if batches and failures == len(batches):
+        # Surface the underlying provider error (e.g. an unsupported/decommissioned
+        # model) instead of a generic message, so the UI can tell the user what to fix.
+        detail = f" — {last_error}" if last_error else ""
+        raise RuntimeError(f"The AI model could not generate FAQs from this content{detail}")
+    return accepted
+
+
+def insert_draft_rows(db: Session, job: FAQGenerationJob, rows: List[FAQ]) -> int:
+    """Shared GROUPING stage tail: bulk-insert the drafted rows."""
+    FAQGenerationJobRepository(db).update_progress(
+        job.id, stage=FAQJobStage.GROUPING, progress_percentage=92.0
+    )
+    if rows:
+        FAQRepository(db).bulk_create(rows)
+    return len(rows)
+
+
+async def run_generation_job(db: Session, job: FAQGenerationJob) -> int:
+    """Execute a GENERATE_ALL / GENERATE_SOURCE job. Returns FAQs created.
+    Status transitions and notifications are the worker's responsibility."""
+    job_repo = FAQGenerationJobRepository(db)
+
+    # Stage 1: resolve sources + model config.
+    job_repo.update_progress(job.id, stage=FAQJobStage.ANALYZING_SOURCES, progress_percentage=5.0)
+    generator = build_generator(db, job.organization_id, job=job)
+
+    knowledge_query = db.query(Knowledge).filter(Knowledge.organization_id == job.organization_id)
+    if job.job_type == FAQJobType.GENERATE_SOURCE.value:
+        knowledge_query = knowledge_query.filter(Knowledge.id == job.knowledge_id)
+    elif job.knowledge_ids:
+        # Explicit selection overrides the skip-already-generated default.
+        knowledge_query = knowledge_query.filter(Knowledge.id.in_(job.knowledge_ids))
+    sources = knowledge_query.all()
+    if not sources:
+        raise ValueError("No knowledge sources to generate from. Add knowledge first.")
+
+    if job.job_type == FAQJobType.GENERATE_ALL.value and not job.knowledge_ids:
+        # Regenerate touches only sources that haven't produced FAQs yet —
+        # deleting a source's FAQs makes it eligible again.
+        generated = FAQRepository(db).knowledge_ids_with_faqs(job.organization_id)
+        sources = [s for s in sources if s.id not in generated]
+        if not sources:
+            raise ValueError(
+                "All knowledge sources already have FAQs. "
+                "Delete a source's FAQs to regenerate it."
+            )
+
+    # Stage 2: read stored page text per source. A broken source (missing
+    # vector table etc.) is skipped, not fatal — unless nothing is readable.
+    job_repo.update_progress(job.id, stage=FAQJobStage.EXTRACTING, progress_percentage=10.0)
+    source_batches: List[Tuple[Knowledge, str]] = []
+    for source in sources:
+        try:
+            # Batch size follows the org's model context window — big-context
+            # models get fewer, larger LLM calls.
+            pages = load_source_pages(db, source, max_chars=generator.batch_chars)
+        except Exception as e:
+            logger.warning(f"Skipping unreadable knowledge source {source.id} ({source.source}): {e}")
+            db.rollback()
+            continue
+        for batch in pack_batches(pages, max_chars=generator.batch_chars):
+            source_batches.append((source, batch))
+    if not source_batches:
+        raise ValueError("The selected knowledge sources contain no readable content.")
+
+    # Stage 3: draft per batch (shared loop).
+    categories = CategoryMerger(db, job.organization_id)
+
+    async def extract(batch: Tuple[Knowledge, str], dedup: DedupState) -> List[GeneratedFAQ]:
+        return await generator.generate_from_text(
+            batch[1],
+            existing_questions=dedup.for_prompt,
+            existing_categories=categories.names,
+        )
+
+    accepted = await draft_batches(db, job, source_batches, extract)
+
+    # Stage 4: group + insert drafts.
+    rows = [
+        FAQ(
+            organization_id=job.organization_id,
+            question=faq.question,
+            answer=faq.answer,
+            category=categories.merge(faq.category),
+            status=FAQStatus.DRAFT,
+            knowledge_id=source.id,
+            source_label=source.source,
+            generation_job_id=job.id,
+            created_by=job.user_id,
+        )
+        for (source, _batch_text), faq in accepted
+    ]
+    return insert_draft_rows(db, job, rows)
+
+
+def maybe_enqueue_auto_faq_job(db: Session, queue_item) -> Optional[FAQGenerationJob]:
+    """Auto-generate draft FAQs when a new knowledge source finishes processing.
+
+    Only fires for orgs that have adopted the feature (a help-center settings
+    row or any FAQ exists), have auto-generate on, are allowed by their plan
+    (cloud), and don't already have an active job for this source. Never raises
+    and always leaves the shared session usable — a hook failure must not fail
+    knowledge processing.
+    """
+    try:
+        from app.repositories.help_center import HelpCenterRepository
+        from app.services.help_center_access import help_center_allowed
+
+        org_id = queue_item.organization_id
+        settings_row = HelpCenterRepository(db).get_by_org(org_id)
+        faq_repo = FAQRepository(db)
+        job_repo_gate = FAQGenerationJobRepository(db)
+        # Adoption gate: auto-generation only after the org explicitly ran a
+        # generate/import (or added an FAQ). A settings row alone is NOT
+        # opt-in — merely opening the admin page creates one.
+        if not job_repo_gate.has_user_initiated_job(org_id) and not faq_repo.exists_for_org(org_id):
+            return None
+        if settings_row and not settings_row.auto_generate:
+            return None
+        if not help_center_allowed(db, org_id):
+            return None
+        knowledge = (
+            db.query(Knowledge)
+            .filter(Knowledge.organization_id == org_id, Knowledge.source == queue_item.source)
+            .order_by(Knowledge.id.desc())  # deterministic: newest row wins on duplicate sources
+            .first()
+        )
+        if not knowledge:
+            return None
+        job_repo = FAQGenerationJobRepository(db)
+        if job_repo.get_active_for_org(org_id, FAQJobType.GENERATE_SOURCE, knowledge_id=knowledge.id):
+            return None
+        from app.services.faq_usage import (
+            count_source_pages,
+            ensure_generation_budget,
+            generation_is_metered,
+            PAGES_PER_CALL,
+        )
+        from math import ceil
+
+        try:
+            pages = count_source_pages(db, knowledge)
+            ensure_generation_budget(db, org_id, max(1, ceil((pages or 1) / PAGES_PER_CALL)))
+        except HTTPException:
+            logger.info(f"Auto FAQ generation skipped for org {org_id}: message budget exhausted")
+            return None
+        job = job_repo.create(
+            FAQGenerationJob(
+                organization_id=org_id,
+                user_id=queue_item.user_id,
+                job_type=FAQJobType.GENERATE_SOURCE.value,
+                knowledge_id=knowledge.id,
+                metered=generation_is_metered(db, org_id),
+            )
+        )
+        logger.info(f"Auto-enqueued FAQ generation job {job.id} for knowledge {knowledge.id}")
+        return job
+    except Exception as e:
+        logger.error(f"Auto FAQ enqueue failed (non-fatal): {e}")
+        # A failed flush would otherwise poison the caller's session and turn
+        # an already-successful knowledge run into a FAILED one.
+        try:
+            db.rollback()
+        except Exception:
+            pass
+        return None