docs: 交付文档、维护指引与端到端回归

补齐 README、开发计划、维护交接与 E2E 用例，执行最终验收。

e2e/services/fake-llm/contract_check.py

@@ -0,0 +1,60 @@
+"""Checks the fake LLM against the backend's real agno code path.
+
+The e2e suite only works if agno accepts what the fake returns. This runs the same
+calls the backend makes — the AI-setup key check and a ChatAgent-style structured
+reply — through agno against a running fake, so an agno upgrade that changes the
+request or response shape fails here with a clear message instead of as flaky e2e
+tests. CI runs it right after the stack starts.
+
+    cd backend && PYTHONPATH=. ENV_FILE=../e2e/.env.e2e NO_ENTERPRISE=1 \\
+        python ../e2e/services/fake-llm/contract_check.py [base_url]
+"""
+
+import asyncio
+import json
+import sys
+import urllib.request
+
+from agno.agent import Agent
+
+from app.models.schemas.chat import ChatResponse
+from app.utils.agno_utils import create_model, test_model_api_key
+
+BASE_URL = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:14010/v1"
+CONTROL = BASE_URL.removesuffix("/v1")
+SCENARIO = "contract-check"
+
+
+def put_scenario() -> None:
+    spec = {"rules": [
+        {"match": {"source": "human", "flags": "i"},
+         "reply": {"message": "Connecting you.", "fields": {"transfer_to_human": True}}},
+        {"reply": {"message": "Happy to help!"}},
+    ]}
+    req = urllib.request.Request(f"{CONTROL}/__scenarios/{SCENARIO}", data=json.dumps(spec).encode(),
+                                 method="PUT", headers={"content-type": "application/json"})
+    urllib.request.urlopen(req).read()
+
+
+async def main() -> int:
+    put_scenario()
+    assert await test_model_api_key("e2e-key", "OPENAI_COMPATIBLE", "e2e-fake", base_url=BASE_URL), \
+        "AI-setup key check failed against the fake"
+
+    model = create_model("OPENAI_COMPATIBLE", "e2e-key", "e2e-fake", max_tokens=1000, base_url=BASE_URL)
+    agent = Agent(model=model, instructions=f"Contract check. [[e2e:{SCENARIO}]]",
+                  response_model=ChatResponse, structured_outputs=True, system_message_role="system")
+
+    handover = (await agent.arun("I want a human")).content
+    assert isinstance(handover, ChatResponse), f"structured reply not parsed: {handover!r}"
+    assert handover.transfer_to_human is True and handover.message == "Connecting you.", handover
+
+    plain = (await agent.arun("hello")).content
+    assert plain.transfer_to_human is False and plain.message == "Happy to help!", plain
+
+    print("fake LLM contract OK")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))

e2e/services/fake-llm/protocol.ts

@@ -0,0 +1,77 @@
+// Wire types shared by the fake LLM server and the test-side client (src/fakes/llm.ts).
+// Type-only: the server imports this with `import type`, so Node's type stripping
+// needs nothing at runtime.
+
+/** A serialisable RegExp. */
+export interface Pattern {
+  source: string
+  flags: string
+}
+
+/**
+ * What the model "says" for one matched turn.
+ *
+ * - `message` / `fields` fill a structured (json_schema) reply — e.g. ChatAgent's
+ *   ChatResponse. `fields` overlays any schema field: transfer_to_human, create_ticket, …
+ * - `text` is used for plain-text requests (the transfer agent, the key check).
+ *   Falls back to `message`.
+ */
+export interface Reply {
+  message?: string
+  fields?: Record<string, unknown>
+  text?: string
+  /** Token counts to report, for metering. Default 1/1. */
+  usage?: { prompt_tokens: number; completion_tokens: number }
+}
+
+/** How the turn goes wrong instead of (or before) answering. */
+export interface Fault {
+  /** Answer with this HTTP status and an OpenAI-style error instead of a completion. */
+  status?: number
+  message?: string
+  /** Wait this long first — past AGENT_RUN_TIMEOUT to exercise the backend's timeout. */
+  delayMs?: number
+  /** Return content that is not the JSON the request's schema asks for. */
+  malformed?: boolean
+}
+
+export interface ToolCall {
+  name: string
+  arguments: Record<string, unknown>
+}
+
+export interface ScenarioRule {
+  /** Tested against the last user message. Omitted = matches anything (otherwise). */
+  match?: Pattern
+  /** Ask the backend to run a tool first; `reply` is sent after the tool result comes back. */
+  toolCall?: ToolCall
+  reply: Reply
+  fault?: Fault
+}
+
+export interface ScenarioSpec {
+  /**
+   * Some backend calls never see the agent's instructions (the transfer agent's prompt
+   * has no `[[e2e:<id>]]` marker), so a scenario can also match on the unique agent name.
+   */
+  agentName?: string
+  rules: ScenarioRule[]
+}
+
+export interface RecordedRequest {
+  at: string
+  system: string
+  lastUser: string
+  lastRole: string
+  /** 'json_schema:<name>' | 'json_object' | null */
+  responseFormat: string | null
+  tools: string[]
+  /** Contents of the tool-result messages in this request — e.g. what knowledge search returned. */
+  toolResults: string[]
+  /** What the fake answered: 'tool_call:<name>' | 'structured' | 'text' | 'error:<status>' | 'malformed' */
+  answeredWith: string
+  /** The model the backend asked for. */
+  model: string
+  /** The last 6 characters of the API key it sent — which key the org's config used. */
+  keySuffix: string
+}

e2e/services/fake-llm/server.ts

@@ -0,0 +1,285 @@
+// A deterministic stand-in for an OpenAI-compatible model, for e2e tests.
+//
+// The backend's agents are pointed here (AI config model_type OPENAI_COMPATIBLE,
+// base_url http://127.0.0.1:14010/v1). Each test registers a *scenario* — scripted
+// replies keyed by a `[[e2e:<id>]]` marker in its agent's instructions or by the
+// agent's unique name — so parallel tests in one org never see each other's script.
+//
+// It speaks only what ChatterMate uses: non-streaming /v1/chat/completions, with or
+// without a strict json_schema response_format (ChatAgent's ChatResponse), and tool
+// calls. Anything else fails loudly instead of returning something plausible.
+//
+// Control API (used by src/fakes/llm.ts):
+//   PUT    /__scenarios/:id            register/replace a ScenarioSpec
+//   GET    /__scenarios/:id/requests   what the backend sent for that scenario
+//   DELETE /__scenarios                forget everything (manual use only: scenario
+//                                      ids are unique per test, and a reset would
+//                                      wipe any other run sharing this fake)
+//
+// An API key starting with `e2e-invalid` is rejected with 401, like a real provider.
+// A rule can also fail on purpose (fault: status, delay, malformed output).
+//
+// Run: node services/fake-llm/server.ts --port 14010   (Node >= 22.18 strips the types)
+
+import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
+import type { Fault, RecordedRequest, Reply, ScenarioRule, ScenarioSpec } from './protocol.ts'
+
+type Json = null | boolean | number | string | Json[] | { [key: string]: Json }
+// JSON Schema is walked generically (properties, $ref, anyOf, …); typing every
+// keyword buys nothing here.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type JsonSchema = { [key: string]: any }
+
+interface ChatMessage {
+  role: string
+  content?: string | Array<{ type: string; text?: string }> | null
+  tool_calls?: unknown[]
+}
+
+interface ChatRequest {
+  model?: string
+  messages: ChatMessage[]
+  stream?: boolean
+  response_format?: { type: string; json_schema?: { name?: string; schema?: JsonSchema } }
+  tools?: Array<{ type: string; function: { name: string } }>
+}
+
+const MARKER = /\[\[e2e:([\w-]+)\]\]/
+const DEFAULT_SCENARIO = '__default'
+const DEFAULT_REPLY: Reply = { message: 'Default reply from the e2e fake LLM.', text: 'OK' }
+const MAX_RECORDED = 50
+// Oldest scenarios are forgotten past this, so a fake that outlives many runs stays small.
+const MAX_SCENARIOS = 5000
+
+const scenarios = new Map<string, ScenarioSpec>()
+const recorded = new Map<string, RecordedRequest[]>()
+
+// ---------- request helpers ----------
+
+function textOf(content: ChatMessage['content']): string {
+  if (typeof content === 'string') return content
+  if (Array.isArray(content)) return content.map((part) => part.text ?? '').join('\n')
+  return ''
+}
+
+function selectScenario(req: ChatRequest): string {
+  const all = req.messages.map((m) => textOf(m.content)).join('\n')
+  const marker = MARKER.exec(all)
+  if (marker && scenarios.has(marker[1])) return marker[1]
+  for (const [id, spec] of scenarios) {
+    if (spec.agentName && all.includes(spec.agentName)) return id
+  }
+  return DEFAULT_SCENARIO
+}
+
+function selectRule(spec: ScenarioSpec | undefined, lastUser: string): ScenarioRule | undefined {
+  if (!spec) return undefined
+  return spec.rules.find((rule) => !rule.match || new RegExp(rule.match.source, rule.match.flags).test(lastUser))
+}
+
+// ---------- structured output ----------
+
+function resolveRef(schema: JsonSchema, root: JsonSchema): JsonSchema {
+  if (typeof schema.$ref !== 'string') return schema
+  const path = schema.$ref.replace(/^#\//, '').split('/')
+  return path.reduce((node: JsonSchema, key: string) => node?.[key], root) ?? {}
+}
+
+function isNullable(schema: JsonSchema): boolean {
+  if (Array.isArray(schema.type)) return schema.type.includes('null')
+  const variants = schema.anyOf ?? schema.oneOf
+  return Array.isArray(variants) && variants.some((v: JsonSchema) => v.type === 'null')
+}
+
+/** The minimal value that satisfies `schema` — the base every structured reply is built on. */
+function defaultFor(rawSchema: JsonSchema, root: JsonSchema): Json {
+  const schema = resolveRef(rawSchema, root)
+  if ('default' in schema) return schema.default as Json
+  if (isNullable(schema)) return null
+  if (Array.isArray(schema.enum)) return schema.enum[0] as Json
+  const variants = schema.anyOf ?? schema.oneOf ?? schema.allOf
+  if (Array.isArray(variants) && variants.length) return defaultFor(variants[0], root)
+  const type = Array.isArray(schema.type) ? schema.type[0] : schema.type
+  switch (type) {
+    case 'string': return ''
+    case 'boolean': return false
+    case 'number':
+    case 'integer': return 0
+    case 'array': return []
+    case 'object':
+    default: {
+      const out: { [key: string]: Json } = {}
+      for (const [key, prop] of Object.entries<JsonSchema>(schema.properties ?? {})) out[key] = defaultFor(prop, root)
+      return out
+    }
+  }
+}
+
+function structuredContent(req: ChatRequest, reply: Reply): string {
+  const schema = req.response_format?.json_schema?.schema
+  const base = schema ? (defaultFor(schema, schema) as { [key: string]: Json }) : {}
+  const message = reply.message ?? DEFAULT_REPLY.message
+  return JSON.stringify({ ...base, message, ...(reply.fields as { [key: string]: Json } | undefined) })
+}
+
+// ---------- completion ----------
+
+function completion(req: ChatRequest, message: { content: string | null; tool_calls?: unknown[] }, usage = { prompt_tokens: 1, completion_tokens: 1 }) {
+  return {
+    id: `chatcmpl-e2e-${Date.now()}`,
+    object: 'chat.completion',
+    created: Math.floor(Date.now() / 1000),
+    model: req.model ?? 'e2e-fake',
+    choices: [{
+      index: 0,
+      message: { role: 'assistant', ...message },
+      finish_reason: message.tool_calls ? 'tool_calls' : 'stop',
+    }],
+    usage: { ...usage, total_tokens: usage.prompt_tokens + usage.completion_tokens },
+  }
+}
+
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
+
+/** A scripted failure: an OpenAI-style error status, or garbage where JSON was asked for. */
+function faulted(req: ChatRequest, fault: Fault, record: RecordedRequest) {
+  if (fault.status) {
+    record.answeredWith = `error:${fault.status}`
+    throw new HttpError(fault.status, fault.message ?? 'The e2e fake LLM failed on purpose.')
+  }
+  record.answeredWith = 'malformed'
+  return completion(req, { content: '{"message": "this reply is cut off' })
+}
+
+async function answer(req: ChatRequest, authorization: string) {
+  const scenarioId = selectScenario(req)
+  const lastUserMsg = [...req.messages].reverse().find((m) => m.role === 'user')
+  const lastUser = textOf(lastUserMsg?.content)
+  const lastRole = req.messages.at(-1)?.role ?? ''
+  const rule = selectRule(scenarios.get(scenarioId), lastUser)
+  const tools = (req.tools ?? []).map((t) => t.function.name)
+  const format = req.response_format
+  const record: RecordedRequest = {
+    at: new Date().toISOString(),
+    system: req.messages.filter((m) => m.role === 'system' || m.role === 'developer').map((m) => textOf(m.content)).join('\n'),
+    lastUser,
+    lastRole,
+    responseFormat: format ? (format.type === 'json_schema' ? `json_schema:${format.json_schema?.name ?? ''}` : format.type) : null,
+    tools,
+    toolResults: req.messages.filter((m) => m.role === 'tool').map((m) => textOf(m.content)),
+    answeredWith: '',
+    model: req.model ?? '',
+    keySuffix: authorization.replace(/^Bearer\s+/i, '').slice(-6),
+  }
+  const list = recorded.get(scenarioId) ?? []
+  list.push(record)
+  recorded.set(scenarioId, list.slice(-MAX_RECORDED))
+
+  if (rule?.fault?.delayMs) await sleep(rule.fault.delayMs)
+  if (rule?.fault && (rule.fault.status || rule.fault.malformed)) return faulted(req, rule.fault, record)
+
+  const usage = rule?.reply.usage
+  let body
+  if (rule?.toolCall && lastRole !== 'tool') {
+    if (!tools.includes(rule.toolCall.name)) {
+      throw new HttpError(422, `scenario '${scenarioId}' wants tool '${rule.toolCall.name}' but the request only offers [${tools.join(', ')}]`)
+    }
+    record.answeredWith = `tool_call:${rule.toolCall.name}`
+    body = completion(req, {
+      content: null,
+      tool_calls: [{
+        id: `call_e2e_${Date.now()}`,
+        type: 'function',
+        function: { name: rule.toolCall.name, arguments: JSON.stringify(rule.toolCall.arguments) },
+      }],
+    })
+  } else if (format && format.type !== 'text') {
+    record.answeredWith = 'structured'
+    body = completion(req, { content: structuredContent(req, rule?.reply ?? DEFAULT_REPLY) }, usage)
+  } else {
+    const reply = rule?.reply ?? DEFAULT_REPLY
+    record.answeredWith = 'text'
+    body = completion(req, { content: reply.text ?? reply.message ?? DEFAULT_REPLY.text! }, usage)
+  }
+  return body
+}
+
+// ---------- http ----------
+
+class HttpError extends Error {
+  status: number
+  constructor(status: number, message: string) {
+    super(message)
+    this.status = status
+  }
+}
+
+async function readJson(req: IncomingMessage): Promise<unknown> {
+  const chunks: Buffer[] = []
+  for await (const chunk of req) chunks.push(chunk as Buffer)
+  const raw = Buffer.concat(chunks).toString('utf8')
+  if (!raw) return {}
+  try {
+    return JSON.parse(raw)
+  } catch {
+    throw new HttpError(400, 'body is not JSON')
+  }
+}
+
+function send(res: ServerResponse, status: number, body: unknown) {
+  res.writeHead(status, { 'content-type': 'application/json' })
+  res.end(JSON.stringify(body))
+}
+
+async function route(req: IncomingMessage, res: ServerResponse) {
+  const url = new URL(req.url ?? '/', 'http://fake-llm')
+  const path = url.pathname.replace(/\/+$/, '')
+  const scenarioPath = /^\/__scenarios\/([\w-]+)(\/requests)?$/.exec(path)
+
+  if (req.method === 'GET' && path === '/health') return send(res, 200, { status: 'ok', scenarios: scenarios.size })
+  if (req.method === 'GET' && path === '/v1/models') return send(res, 200, { object: 'list', data: [{ id: 'e2e-fake', object: 'model' }] })
+
+  if (req.method === 'POST' && path === '/v1/chat/completions') {
+    // Lets a test exercise the "invalid API key" path: the backend live-tests keys.
+    if (/^Bearer e2e-invalid/.test(req.headers.authorization ?? '')) {
+      throw new HttpError(401, 'Incorrect API key provided.')
+    }
+    const body = (await readJson(req)) as ChatRequest
+    if (body.stream) throw new HttpError(400, 'the e2e fake LLM does not stream — ChatterMate never asks it to; if it does now, extend services/fake-llm')
+    if (!Array.isArray(body.messages)) throw new HttpError(400, 'messages[] is required')
+    return send(res, 200, await answer(body, req.headers.authorization ?? ''))
+  }
+
+  if (scenarioPath && req.method === 'PUT' && !scenarioPath[2]) {
+    const spec = (await readJson(req)) as ScenarioSpec
+    if (!Array.isArray(spec.rules)) throw new HttpError(400, 'rules[] is required')
+    scenarios.set(scenarioPath[1], spec)
+    recorded.delete(scenarioPath[1])
+    while (scenarios.size > MAX_SCENARIOS) {
+      const oldest = scenarios.keys().next().value!
+      scenarios.delete(oldest)
+      recorded.delete(oldest)
+    }
+    return send(res, 200, { ok: true })
+  }
+  if (scenarioPath && req.method === 'GET' && scenarioPath[2]) return send(res, 200, recorded.get(scenarioPath[1]) ?? [])
+  if (req.method === 'DELETE' && path === '/__scenarios') {
+    scenarios.clear()
+    recorded.clear()
+    return send(res, 200, { ok: true })
+  }
+
+  throw new HttpError(404, `no route for ${req.method} ${url.pathname}`)
+}
+
+const portArg = process.argv.indexOf('--port')
+const port = Number(portArg > -1 ? process.argv[portArg + 1] : process.env.FAKE_LLM_PORT ?? 14010)
+
+createServer((req, res) => {
+  route(req, res).catch((err: unknown) => {
+    const status = err instanceof HttpError ? err.status : 500
+    const message = err instanceof Error ? err.message : String(err)
+    console.error(`[fake-llm] ${req.method} ${req.url} -> ${status}: ${message}`)
+    send(res, status, { error: { message, type: 'e2e_fake_llm_error' } })
+  })
+}).listen(port, '127.0.0.1', () => console.log(`[fake-llm] listening on http://127.0.0.1:${port}`))