feat(modules): 帮助中心、工单、工作流与多渠道接入

补齐 FAQ、帮助中心、工单、可视化工作流及 Telegram/WhatsApp/Slack 等渠道模块。

backend/app/services/ticket_investigation.py

@@ -0,0 +1,349 @@
+"""
+Copyright 2024-2026 ChatterMate
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Phase 3 investigation engine: hypothesis-driven root-cause analysis with every
+tool call captured as a first-class InvestigationEvent row (the glass box).
+Orchestrated here, executed by the ticket_investigator worker.
+"""
+
+import asyncio
+import json
+import time
+from typing import List, Optional, Tuple
+from uuid import UUID
+
+from app.core.logger import get_logger
+from app.database import SessionLocal
+from app.models.investigation import (
+    HypothesisStatus,
+    InvestigationEvent,
+    InvestigationEventType,
+    InvestigationHypothesis,
+    InvestigationRun,
+    InvestigationRunStatus,
+    RCADocument,
+)
+from app.models.ticket import Ticket
+from app.utils.guardrails import GuardrailAction, PIIDetector
+
+logger = get_logger(__name__)
+
+# Per-hypothesis tool budget; the run-level max_tool_calls caps the total.
+HYPOTHESIS_TOOL_CALL_LIMIT = 6
+# Truncation applied at capture time — raw payloads never land in the DB.
+MAX_EVENT_INPUT_CHARS = 2000
+MAX_EVENT_RESULT_CHARS = 4000
+# Bounded time for the final RCA synthesis (outside the main wall clock so a
+# timed-out investigation can still produce a partial RCA).
+RCA_SYNTHESIS_TIMEOUT_SECONDS = 180
+
+VALID_VERDICT_STATUSES = {
+    HypothesisStatus.VALIDATED.value,
+    HypothesisStatus.INVALIDATED.value,
+    HypothesisStatus.INCONCLUSIVE.value,
+}
+
+
+def redact_snippet(text: str, max_chars: int) -> str:
+    """PII-redact and truncate a tool payload before it is persisted."""
+    if not text:
+        return ""
+    result = PIIDetector.detect(text[: max_chars * 2], action=GuardrailAction.REDACT)
+    redacted = result.redacted_text if result.redacted_text is not None else text
+    redacted = redacted[:max_chars]
+    return redacted
+
+
+class EvidenceRecorder:
+    """Writes InvestigationEvent rows as the investigator works.
+
+    Each write uses its own short-lived session so evidence is visible live
+    (the glass-box UI polls during active runs) and survives a failed run.
+    Also the run's tool-call meter: the budget check reads self.tool_calls.
+    """
+
+    def __init__(self, run_id: UUID, ticket_id: UUID):
+        self.run_id = run_id
+        self.ticket_id = ticket_id
+        self.seq = 0
+        self.tool_calls = 0
+        self.hypothesis_id: Optional[UUID] = None
+        # Per-hypothesis tool-call outcomes: {hypothesis_id: [total, errored]}.
+        # A verdict is only worth as much as the evidence under it, and a run
+        # where every query failed must not be able to assert a fact (#318).
+        self.calls_by_hypothesis: dict = {}
+        # Run-level totals, for the dashboard's connector banner.
+        self.failed_tool_calls = 0
+        # tool function name -> configured connector (MCP server) name.
+        self.connector_by_function: dict = {}
+        # In-memory digest for RCA synthesis, mirroring the persisted rows.
+        self.evidence_lines: List[str] = []
+
+    def map_connectors(self, mcp_tools: list) -> None:
+        for tool in mcp_tools or []:
+            connector = getattr(tool, "_connector_name", None) or "MCP"
+            for function_name in getattr(tool, "functions", {}) or {}:
+                self.connector_by_function[function_name] = connector
+
+    def _write(self, **fields) -> None:
+        self.seq += 1
+        try:
+            with SessionLocal() as db:
+                db.add(
+                    InvestigationEvent(
+                        run_id=self.run_id,
+                        ticket_id=self.ticket_id,
+                        hypothesis_id=self.hypothesis_id,
+                        seq=self.seq,
+                        **fields,
+                    )
+                )
+                db.commit()
+        except Exception as e:
+            logger.error(f"Failed to record investigation event: {e}")
+
+    def record_phase(self, label: str) -> None:
+        """Phase marker — powers the live ticker line in the UI."""
+        self._write(event_type=InvestigationEventType.PHASE, label=label[:300])
+
+    async def tool_hook(self, function_name: str, function, arguments: dict):
+        """agno tool hook: times the call and captures redacted input/result.
+        Registered on the hypothesis-testing agents only."""
+        # The Groq structured-output capture tool is plumbing, not evidence.
+        if function_name == "json":
+            return await function(**arguments)
+        started = time.monotonic()
+        error: Optional[str] = None
+        result = None
+        try:
+            result = await function(**arguments)
+            return result
+        except Exception as e:
+            error = str(e)
+            raise
+        finally:
+            self.tool_calls += 1
+            if error:
+                self.failed_tool_calls += 1
+            if self.hypothesis_id is not None:
+                tally = self.calls_by_hypothesis.setdefault(self.hypothesis_id, [0, 0])
+                tally[0] += 1
+                if error:
+                    tally[1] += 1
+            duration_ms = int((time.monotonic() - started) * 1000)
+            try:
+                input_text = json.dumps(arguments, default=str)
+            except Exception:
+                input_text = str(arguments)
+            input_snippet = redact_snippet(input_text, MAX_EVENT_INPUT_CHARS)
+            result_snippet = redact_snippet(str(result), MAX_EVENT_RESULT_CHARS) if result is not None else None
+            self._write(
+                event_type=InvestigationEventType.TOOL_CALL,
+                tool_name=function_name[:200],
+                connector_name=self.connector_by_function.get(function_name, "MCP")[:200],
+                tool_input=input_snippet,
+                tool_result=result_snippet,
+                duration_ms=duration_ms,
+                error=error[:2000] if error else None,
+            )
+            self.evidence_lines.append(
+                f"[E{self.seq}] {function_name}({input_snippet[:300]}) -> "
+                + (f"ERROR: {error[:200]}" if error else (result_snippet or "")[:500])
+            )
+
+
+    def tool_outcomes(self, hypothesis_id: UUID) -> tuple:
+        """(total, errored) tool calls made while testing this hypothesis."""
+        total, errored = self.calls_by_hypothesis.get(hypothesis_id, [0, 0])
+        return total, errored
+
+
+def enforce_evidence_backed_verdict(
+    hypothesis: InvestigationHypothesis, recorder: "EvidenceRecorder"
+) -> bool:
+    """Downgrade a definite verdict that no successful query supports.
+
+    The model returns status and confidence as free text, and nothing else
+    checks them against what actually happened. A run whose every search
+    errored — a missing index, a rejected client version — can still come back
+    "the record does not exist, validated, 0.90", which a support agent would
+    reasonably act on. Tool errors are the absence of evidence, never evidence
+    of absence.
+
+    Only fires when the phase made tool calls and every one of them failed;
+    a hypothesis reasoned from ticket context alone is left as the model
+    returned it. Returns True when it changed the verdict.
+    """
+    total, errored = recorder.tool_outcomes(hypothesis.id)
+    if not total or errored < total:
+        return False
+    if str(hypothesis.status) == HypothesisStatus.INCONCLUSIVE.value:
+        return False
+
+    original = str(hypothesis.status)
+    confidence = hypothesis.confidence
+    hypothesis.status = HypothesisStatus.INCONCLUSIVE
+    hypothesis.confidence = 0.0
+    # The model's own conclusion is dropped rather than kept: it reads as a
+    # finding, and the RCA writer would quote it as one.
+    hypothesis.conclusion = (
+        f"Not tested — all {total} tool call{'' if total == 1 else 's'} in this phase "
+        f"failed, so no evidence was gathered. The tester reported "
+        f"\"{original}\"{f' at {confidence:.2f} confidence' if confidence is not None else ''}, "
+        "which nothing supports. Check the connector errors in the evidence log."
+    )
+    return True
+
+
+def hypotheses_digest(hypotheses: List[InvestigationHypothesis]) -> str:
+    lines = []
+    for h in hypotheses:
+        lines.append(
+            f"H{h.idx}: {h.title} — {h.status} (confidence {h.confidence if h.confidence is not None else 'n/a'})\n"
+            f"    {h.conclusion or h.rationale or ''}"
+        )
+    return "\n".join(lines)
+
+
+async def run_investigation_phases(
+    db,
+    run: InvestigationRun,
+    ticket: Ticket,
+    agent,
+    context_message: str,
+    mcp_tools: list,
+    recorder: EvidenceRecorder,
+) -> Tuple[List[InvestigationHypothesis], bool]:
+    """Generate hypotheses, then test each within the run's tool budget.
+    Mutates hypothesis rows (committed per step so the UI streams progress).
+    Returns (hypotheses, budget_exhausted)."""
+    recorder.record_phase("Generating hypotheses")
+    plan = await agent.generate_hypotheses(context_message)
+    if plan is None or not plan.hypotheses:
+        raise RuntimeError("Hypothesis generation produced no result")
+
+    hypotheses: List[InvestigationHypothesis] = []
+    for i, spec in enumerate(plan.hypotheses, start=1):
+        hypothesis = InvestigationHypothesis(
+            run_id=run.id,
+            ticket_id=ticket.id,
+            idx=i,
+            title=spec.title[:300],
+            rationale=spec.rationale,
+            status=HypothesisStatus.PENDING,
+        )
+        db.add(hypothesis)
+        hypotheses.append(hypothesis)
+    db.commit()
+
+    budget_exhausted = False
+    max_tool_calls = run.max_tool_calls if run.max_tool_calls is not None else 25
+    for hypothesis in hypotheses:
+        remaining_budget = max(0, max_tool_calls - recorder.tool_calls)
+        if remaining_budget <= 0 and mcp_tools:
+            budget_exhausted = True
+            hypothesis.status = HypothesisStatus.INCONCLUSIVE
+            hypothesis.confidence = 0.0
+            hypothesis.conclusion = "Not tested — the run's tool-call budget was exhausted."
+            db.commit()
+            continue
+
+        recorder.hypothesis_id = hypothesis.id
+        recorder.record_phase(f"Testing H{hypothesis.idx}: {hypothesis.title}")
+        hypothesis.status = HypothesisStatus.TESTING
+        db.commit()
+
+        verdict = await agent.test_hypothesis(
+            context_message,
+            hypothesis.title,
+            hypothesis.rationale,
+            mcp_tools=mcp_tools,
+            tool_call_limit=min(HYPOTHESIS_TOOL_CALL_LIMIT, remaining_budget) if mcp_tools else None,
+            tool_hooks=[recorder.tool_hook],
+        )
+        if verdict is not None and verdict.status in VALID_VERDICT_STATUSES:
+            hypothesis.status = verdict.status
+            hypothesis.confidence = max(0.0, min(1.0, verdict.confidence))
+            hypothesis.conclusion = verdict.conclusion
+            if enforce_evidence_backed_verdict(hypothesis, recorder):
+                logger.warning(
+                    f"H{hypothesis.idx} claimed '{verdict.status}' with every tool "
+                    f"call failing; forced to inconclusive"
+                )
+        else:
+            hypothesis.status = HypothesisStatus.INCONCLUSIVE
+            hypothesis.confidence = 0.0
+            # A provider that refused the request is a hard, deterministic
+            # failure — saying "no parseable verdict" for it reads as a model
+            # quality problem and hides a fixable connector issue (#303).
+            hypothesis.conclusion = (
+                getattr(agent, "last_provider_error", None)
+                or "The tester produced no parseable verdict."
+            )
+        db.commit()
+    recorder.hypothesis_id = None
+    return hypotheses, budget_exhausted
+
+
+async def synthesize_and_store_rca(
+    db,
+    run: InvestigationRun,
+    ticket: Ticket,
+    agent,
+    context_message: str,
+    hypotheses: List[InvestigationHypothesis],
+    recorder: EvidenceRecorder,
+    partial: bool,
+) -> Optional[RCADocument]:
+    """Write the versioned RCA document from whatever evidence exists."""
+    from app.repositories.ticket import InvestigationRepository
+
+    recorder.record_phase("Writing root-cause analysis")
+    try:
+        result = await asyncio.wait_for(
+            agent.synthesize_rca(
+                context_message,
+                hypotheses_digest(hypotheses),
+                "\n".join(recorder.evidence_lines[-60:]),
+                partial=partial,
+            ),
+            timeout=RCA_SYNTHESIS_TIMEOUT_SECONDS,
+        )
+    except asyncio.TimeoutError:
+        logger.error(f"RCA synthesis timed out for {ticket.display_number}")
+        return None
+    if result is None:
+        return None
+
+    rca = RCADocument(
+        ticket_id=ticket.id,
+        run_id=run.id,
+        version=InvestigationRepository(db).next_rca_version(ticket.id),
+        summary=result.summary,
+        impact=result.impact,
+        timeline=[entry.model_dump() for entry in result.timeline],
+        investigation_log=result.investigation_log,
+        contributing_factors=result.contributing_factors,
+        conclusion=result.conclusion,
+        remediation=result.remediation,
+        prevention=result.prevention,
+        customer_summary=result.customer_summary,
+        confidence=max(0.0, min(1.0, result.confidence)),
+        is_partial=partial,
+        generated_by="ai",
+    )
+    db.add(rca)
+    db.commit()
+    return rca