WangShen007/replypier
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: 交付文档、维护指引与端到端回归

Browse Source 
补齐 README、开发计划、维护交接与 E2E 用例，执行最终验收。
This commit is contained in:
WangShen007 committed 2026-10-21 21:40:00 +08:00
1 parent 5be5ca3376
commit 308332eb94
216 files changed
+37038
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

e2e/.env.e2e
+79
View File
@@ -0,0 +1,79 @@
# Backend env for the e2e stack. scripts/stack.sh passes this file as ENV_FILE,
# so the backend reads ONLY this — never a developer's backend/.env.
# Every value here is a throwaway for local/CI test runs; nothing is a real secret.
# NO_ENTERPRISE and the real DATABASE_URL are exported by stack.sh (they depend on
# the edition and the local Postgres). This placeholder points nowhere, so a backend
# started without stack.sh fails fast instead of falling back to the dev database.
DATABASE_URL=postgresql+psycopg://set-by-stack-sh@127.0.0.1:1/chattermate_e2e
ENVIRONMENT=test
JWT_SECRET_KEY=e2e-jwt-secret-not-for-production
CONVERSATION_SECRET_KEY=e2e-conversation-secret-not-for-production
# Same fixed test-only key as backend/tests/conftest.py.
ENCRYPTION_KEY=QmFTbXc5RWQ4czRfQWpqSnhqZjhraGtYSGFYaXZ4SkRNS2kxZFB1Y2NrMD0=
FRONTEND_URL=http://127.0.0.1:15173
BACKEND_URL=http://127.0.0.1:18000
VITE_WIDGET_URL=http://127.0.0.1:18000
# Dashboard + the fake customer site the widget is embedded on (see src/components/chat-widget.ts).
CORS_ORIGINS=["http://127.0.0.1:15173","http://127.0.0.1:13000"]
HELP_CENTER_PUBLIC_MODE=path
# Redis off: no shared cors:origins cache, no dev Redis traffic.
REDIS_ENABLED=false
# All mail goes to the stack's Mailpit (scripts/stack.sh). No username means no
# SMTP login, which is how Mailpit runs.
SMTP_SERVER=127.0.0.1
SMTP_PORT=11025
SMTP_USERNAME=
SMTP_PASSWORD=
FROM_EMAIL=no-reply@e2e.example.com
# No push notifications, no S3.
FIREBASE_CREDENTIALS=/nonexistent/firebase-config.json
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
# Providers that ignore the org's base_url (e.g. the CHATTERMATE default model)
# still land on the fake LLM through the openai SDK's env fallback.
OPENAI_BASE_URL=http://127.0.0.1:14010/v1
OPENAI_API_KEY=e2e-fake-key
# Short enough that the "AI times out" journey runs quickly; src/support/timeouts.ts waits longer.
AGENT_RUN_TIMEOUT=15
TOKENIZERS_PARALLELISM=false
# Third-party APIs → the recording fake (services/fake-externals). See
# backend/app/core/external_urls.py; deployments never set these.
TELEGRAM_API_BASE=http://127.0.0.1:14020/telegram
META_GRAPH_BASE=http://127.0.0.1:14020/graph
META_GRAPH_INSTAGRAM_BASE=http://127.0.0.1:14020/graph-instagram
SLACK_API_BASE=http://127.0.0.1:14020/slack
LINE_API_BASE=http://127.0.0.1:14020/line
TWILIO_API_BASE=http://127.0.0.1:14020/twilio
PLIVO_API_BASE=http://127.0.0.1:14020/plivo
HUBSPOT_API_BASE=http://127.0.0.1:14020/hubspot
PIPEDRIVE_OAUTH_BASE=http://127.0.0.1:14020/pipedrive-oauth
# Pipedrive names each company's API host in its token response; the fake names itself.
PIPEDRIVE_API_BASE=http://127.0.0.1:14020/pipedrive
JIRA_AUTH_BASE=http://127.0.0.1:14020/jira-auth
JIRA_API_BASE=http://127.0.0.1:14020/jira
SHOPIFY_ADMIN_BASE=http://127.0.0.1:14020/shopify
# Webhook signing secrets the channel drivers sign with (src/channels/*). Must match
# env.channelSecrets in src/env.ts.
META_APP_SECRET=e2e-meta-app-secret
SLACK_SIGNING_SECRET=e2e-slack-signing-secret
SLACK_CLIENT_ID=e2e-slack-client-id
SLACK_CLIENT_SECRET=e2e-slack-client-secret
HUBSPOT_CLIENT_ID=e2e-hubspot-client-id
HUBSPOT_CLIENT_SECRET=e2e-hubspot-client-secret
PIPEDRIVE_CLIENT_ID=e2e-pipedrive-client-id
PIPEDRIVE_CLIENT_SECRET=e2e-pipedrive-client-secret
JIRA_CLIENT_ID=e2e-jira-client-id
JIRA_CLIENT_SECRET=e2e-jira-client-secret
JIRA_REDIRECT_URI=http://127.0.0.1:18000/api/v1/jira/callback
# Shopify signs the embedded app's session tokens with the app secret (src/api/shopify.ts).
SHOPIFY_API_KEY=e2e-shopify-api-key
SHOPIFY_API_SECRET=e2e-shopify-api-secret
e2e/.gitignore
+13
View File
@@ -0,0 +1,13 @@
node_modules/
test-results/
playwright-report/
blob-report/
.auth/
.cache/
.stack/
.schemathesis/
test-results-quarantine/
playwright-report-quarantine/
# Screenshot baselines are made on Linux in CI only.
specs/**/*-snapshots/*-darwin.png
specs/**/*-snapshots/*-win32.png
e2e/AUTHORING.md
+171
View File
@@ -0,0 +1,171 @@
# ChatterMate end-to-end tests
Playwright tests that drive the real dashboard, the real widget and the real backend.
Each run gets its own stack:
- a throwaway Postgres database
- the backend
- a production build of the dashboard
- a fake LLM
- a pretend customer site
All of it runs on `127.0.0.1` ports that never touch a dev setup on `localhost`.
Unit tests (`backend/tests`, `frontend` vitest) still cover logic and edge cases. These tests cover **journeys**: what a customer or an agent actually does.
## Running
```bash
cd e2e
npm ci
npx playwright install chromium
npm test # starts the stack if needed, runs everything
npm run test:smoke # just @smoke
npx playwright test specs/handover --headed # one area, watch it
npm run test:ui # Playwright UI mode
npm run report # last HTML report (traces on failure)
```
**Prerequisites:**
- A local Postgres with pgvector, reachable as `postgresql://$USER@127.0.0.1:5432` (set `E2E_PG_URL` otherwise). The user must be able to create databases and the `vector` extension.
- `backend/venv` with the backend requirements (or set `E2E_PYTHON`).
- `frontend/node_modules` (or set `E2E_FRONTEND_NODE_MODULES`).
- [`mailpit`](https://mailpit.axllent.org) on the PATH (`brew install mailpit`). Every email the backend sends lands there.
**Faster reruns:** keep the stack up between runs.
```bash
./scripts/stack.sh up # start everything once
npx playwright test # reuses the running stack
./scripts/stack.sh reset # fresh database
./scripts/stack.sh logs backend
./scripts/stack.sh down
```
**Safety:**
- `stack.sh` only ever manages `chattermate_e2e` and its template `chattermate_e2e_tpl`. It refuses any database name that does not end in `_e2e`.
- The backend reads only `e2e/.env.e2e` (through `ENV_FILE`), never `backend/.env`.
- The OSS edition runs with `NO_ENTERPRISE=1`, and the dashboard is built without `src/modules/enterprise`, so it behaves like a community install even in a checkout that has the submodules.
## Editions
| | OSS (default) | Enterprise (`E2E_EDITION=enterprise`) |
|---|---|---|
| Needs | nothing extra | the enterprise submodules checked out |
| Backend | `NO_ENTERPRISE=1`, so community behaviour | enterprise module loaded, its migrations and seed applied |
| Setup project | the first-run form (`specs/setup/`) | real signup with the emailed code (private suite) |
| Specs | `specs/**` | the private suite's specs (`--project=enterprise`) **and** every OSS spec (`--project=chromium`) |
Both setups write the same org contract (`src/org-state.ts`) through `prepareOrg()`, so an OSS spec never needs to know which edition it runs on.
The private suite lives in the enterprise frontend module's `e2e/` folder. It imports only `@e2e`, and nothing from it belongs in this repo.
## How the code is organised
```
specs/<feature>/*.spec.ts what the product does — no selectors, no raw HTTP
src/pages/ one class per dashboard screen; locators live ONLY here…
src/components/ …or here, for UI shared across screens (chat-widget.ts)
src/api/ typed seeding calls, one module per backend area
src/fakes/llm.ts scripts what the AI says (see below)
src/fixtures/ wires it together: agent, widget, inbox, llm, adminApi, org…
src/index.ts the only import specs use: `import { test, expect } from '@e2e'`
specs/setup/*.setup.ts runs once per run: org, AI config, support team, signed-in states
```
| When this changes… | …edit only this |
|---|---|
| A label or the DOM of a screen | that screen's page object (or the component) |
| An API route or payload | the module in `src/api/` |
| What a default test agent looks like | `src/fixtures/index.ts` |
| The model reply format | `services/fake-llm` (and run `contract_check.py`) |
## Rules
1. **Locators:** `getByRole` / `getByLabel` first, then `getByText` / `getByPlaceholder`. CSS only inside `src/pages` or `src/components`, with a comment saying why (usually "no accessible name yet"). When you need one, prefer adding an `aria-label` to the app instead.
2. **Web-first assertions only:** `await expect(locator).toHaveText(…)`. Never use `waitForTimeout`, and never `expect(await x.isVisible())`. Lint enforces both.
3. **Seed through the API, assert through the UI.** Use the UI for setup only when the setup *is* the feature under test.
4. **Every test is independent.** Fixtures give each test its own agent and widget with unique names. Never rely on another test's data or on test order.
5. **Org-wide settings** (AI config, business hours, ticket settings) are changed only in `*.org.spec.ts`. Those run in a separate `--workers=1` invocation (`npm run test:org`).
6. **Page objects don't assert,** except to wait for their own page to load. Assertions live in specs.
7. **Wrap user-level actions in `step()`** (`src/support/step.ts`), so reports read like a story.
8. **Tags in the title:**
- `@smoke` runs on every PR.
- `@<feature>` is the folder name.
- `@slow` marks a slow test.
- `@quarantine` excludes a test from required runs; it must link an issue.
9. **Never import `@playwright/test` in a spec.** Import `@e2e` instead, so every spec gets the shared fixtures. This is also what keeps the enterprise suite on a single Playwright copy.
## Scripting the AI
Each test's agent carries a hidden marker. The fake LLM uses it to find that test's script, so parallel tests never see each other's replies.
```ts
await llm.script(agent, (s) => s
.when(/refund/i).handOver('Let me get a teammate.') // transfer_to_human
.when(/broken/i).createTicket('I have logged it.', 'Widget broken') // runs the create_ticket tool
.when(/@/).captureLead('Thanks!', { email, name }) // request_lead_capture + lead_*
.when(/thanks/i).endChat('Goodbye!') // end_chat (+ rating if the agent asks)
.when(/pricing/i).callTool('search_knowledge_base', { query: 'pricing' }).thenReply('Plans start at $10.')
.otherwise().reply('Happy to help!'))
const sent = await llm.requests(agent) // what the backend sent the model: system, lastUser, toolResults…
```
- **Reply format:** replies are structured `ChatResponse` JSON, the same as with a real model. Handover, end-chat and lead capture are fields of that JSON. Tickets are a real tool call (`create_ticket`); the `create_ticket` JSON field alone creates nothing. `reply(message, fields)` sets any other field.
- **Pace conversations like a person:** wait for the agent's reply before the visitor's next message. Replies to back-to-back messages are produced concurrently and can arrive in either order.
- **Failures on purpose:** `.when(/x/).fail(500)`, `.after(25_000).reply(…)` (past `AGENT_RUN_TIMEOUT`), `.malformed()`, and `.costing(promptTokens, completionTokens)` for metering.
- **Contract check:** `services/fake-llm/contract_check.py` runs agno itself against the fake. Run it after upgrading agno.
## Channels and other third parties
The backend's calls to Telegram, Meta, Slack, LINE, SMS providers, HubSpot, Pipedrive, Jira and Shopify go to `services/fake-externals`. The `*_API_BASE` (and `PIPEDRIVE_OAUTH_BASE`, `JIRA_AUTH_BASE`, `SHOPIFY_ADMIN_BASE`) overrides in `.env.e2e` point them there; production never sets these. Email replies go over SMTP into Mailpit.
The fake records every call and answers like the provider would. Use `FakeExternals` to assert on what the backend sent.
Each channel has a driver in `src/channels/` that plays the provider:
- `connect` gives every test its own account, routed to the test agent
- `chatWith(name).send(text)` delivers a signed webhook exactly as the provider would (the signing secrets match `.env.e2e`)
- `replies()` reads what the business sent back, from the fake or Mailpit; channels with buttons also have `buttons()` and `tap()`
A journey that should hold on every channel loops over `channels` (or `channelsWithButtons`) from `src/channels/index.ts`. Adding a channel is one driver file and one line in that registry.
- **Replies are attributed:** a driver only counts what the backend sent *as that account* (its token, number or path), so a reply leaking from another account fails.
- **Provider limits are real:** the fake rejects payloads past each provider's published limits (`services/fake-externals/limits.ts`), the way the provider would.
- **Faults:** `new FakeExternals(request).fail({ provider, action, contains, status, body, times })` makes matching calls fail; scope it with `contains` (e.g. the customer id) so parallel tests are untouched.
- **MCP:** `services/fake-mcp` is a real MCP server (streamable HTTP) with a `lookup_order` tool.
## Fixtures worth knowing
- `customerPage` / `widget`: a visitor's browser, never signed in to the dashboard.
- `makeAgent(settings, groupIds)`, `makeMember({ role, groupIds, seeAllAiChats })`: more agents, and teammates signed in on their own request context.
- `handoverTeam` / `inbox`: each handover test gets its own teammate and group.
- `diagnostics` (automatic): a failed test gets the backend log, model requests and third-party calls attached.
- `expectUnchanged(what, read)`: the "and nothing else happens" check (the AI stays quiet, a forged webhook gets no reply).
## More than the default run
| What | How | Where it runs |
|---|---|---|
| Production-like backend | `E2E_REDIS_URL=redis://127.0.0.1:6379/5 E2E_BACKEND_WORKERS=2 ./scripts/stack.sh up`; enables `@redis` tests | nightly |
| Widget on Firefox, WebKit, phones | `E2E_MORE_BROWSERS=1 npx playwright test --project=widget-firefox …` | nightly |
| Visual | `npx playwright test --project=visual`; baselines are Linux-only, made by the nightly job (or a manual run with *update snapshots*) | nightly |
| Accessibility | `specs/a11y`; new serious findings fail; after fixing some, refresh with `UPDATE_A11Y_BASELINE=1` | every run |
| Size and speed budgets | `specs/perf` | every run |
| Migrations | `./scripts/upgrade-check.sh [ref]`: the last release's schema upgraded to HEAD must equal a fresh install | nightly |
| API fuzzing | `./scripts/api-fuzz.sh` (Schemathesis): every read endpoint, no 500s | nightly |
## Adding a test
1. Find or create `specs/<feature>/`, and add a row to `FEATURES.md`.
2. Write the spec in product language, using the fixtures (`agent`, `widget`, `inbox`, `llm`, `loginPage`, …).
3. If a screen isn't covered yet, add a page object in `src/pages/`. If seeding needs a new endpoint, add it to the module in `src/api/`.
4. `npx playwright test specs/<feature> --repeat-each=10` must pass before you open a PR.
5. Run `npm run lint && npm run typecheck`.
## Troubleshooting
- **Port already in use:** something else is on 18000, 15173, 14010, 14020, 14030, 13000, 11025 or 18025. Run `./scripts/stack.sh down`, or stop the other process.
- **The launcher never becomes clickable:** the suite runs with `reducedMotion: 'reduce'` because the launcher floats forever otherwise. Keep that option on any new browser context.
- **The customer site must be a real server:** Chrome blocks a page faked with `page.route` from calling `127.0.0.1` (Local Network Access).
- **The widget and loader are built from source on every run.** `stack.sh` builds the widget bundle and `chattermate.min.js` from the current `frontend/src/webclient` into `e2e/.cache`, and points the backend at that build with `WIDGET_ASSETS_DIR`. The committed production bundles in `backend/assets` and `frontend/public/webclient` are never rebuilt or modified.
e2e/FEATURES.md
+73
View File
@@ -0,0 +1,73 @@
# Feature coverage (OSS)
One row per user journey. Every feature PR adds or updates its row and the spec that covers it.
- **Priority:** P1 is the first waves, P2 comes later, N runs nightly only.
- **Status:** ✅ covered · 🚧 in progress · ⬜ not yet
| Feature | Journey | Spec | Pri | Status |
|---|---|---|---|---|
| Setup and auth | First-run setup creates the org and lands on the dashboard | `specs/setup/org.setup.ts` | P1 | ✅ |
| Setup and auth | Admin signs in | `specs/auth/login.spec.ts` | P1 | ✅ |
| Setup and auth | Wrong password shows an error | `specs/auth/login.spec.ts` | P1 | ✅ |
| Setup and auth | Sign out | `specs/auth/login.spec.ts` | P1 | ✅ |
| Setup and auth | Agent role is denied AI settings | `specs/auth/login.spec.ts` | P1 | ✅ |
| Setup and auth | Sign-out, deactivation and a password change end sessions server-side; a refresh token is not an access token | `specs/auth/sessions.spec.ts` | P1 | ✅ |
| Team | The Agent role is refused every admin API; sees channels without secrets; assigned-only sees just its group's handovers | `specs/team/permissions.spec.ts` | P1 | ✅ |
| Widget chat | Customer gets the AI answer | `specs/widget/widget-chat.spec.ts` | P1 | ✅ |
| Widget chat | History survives a reload | `specs/widget/widget-chat.spec.ts` | P1 | ✅ |
| Widget chat | The dashboard's iframe snippet chats (its socket comes from the API's own origin) | `specs/widget/iframe-snippet.spec.ts` | P1 | ✅ |
| Widget chat | First authorization answer lost (socket gets the anonymous token) → widget authorizes again and chats | `specs/widget/widget-chat.spec.ts` | P1 | ✅ |
| Widget chat | Email-first agent (gate → chat, person recorded), Ask anything panel, lead capture with no email asked — each also with the first authorization lost | `specs/widget/chat-styles.spec.ts` | P1 | ✅ |
| Widget chat | Agent wraps up → customer rates the chat → rating stored against the agent | `specs/widget/rating.spec.ts` | P1 | ✅ |
| Widget chat | A second browser that types a customer's email gets its own empty conversation (the owner keeps theirs), can't rename them or make them a lead, and an address claimed before its owner signs in yields none of their chats | `specs/widget/conversation-ownership.spec.ts` | P1 | ✅ |
| Widget chat | Scoping keeps a visitor's own conversation: a second tab joins it; a visitor the site signed in finds it in another browser, and after `ChatterMate.identify()` | `specs/widget/conversation-ownership.spec.ts` | P1 | ✅ |
| Widget chat | Provider error, provider timeout and malformed reply each end in an apology; the next message works | `specs/widget/ai-failures.spec.ts` | P1 | ✅ |
| Human handover | AI hands over → contact form → agent takes over → two-way messages → the AI stays silent | `specs/handover/handover.spec.ts` | P1 | ✅ |
| Human handover | After hours, or nobody online: the chat waits in the team's queue with a reply that says so | `specs/handover/after-hours.org.spec.ts`, `nobody-online.spec.ts` | P1 | ✅ |
| Human handover | A visitor's attachment reaches the teammate through a signed link | `specs/handover/attachments.spec.ts` | P2 | ✅ |
| Agents | Create in the dashboard → edit instructions → they reach the model | `specs/agents/agent-editor.spec.ts` | P1 | ✅ |
| Agents | Toggles (handover, rating, reply buttons) | — | P1 | ⬜ |
| Knowledge | Add text → answer is grounded in it (real search) | `specs/knowledge/knowledge-answer.spec.ts` | P1 | ✅ |
| Knowledge | Deleted knowledge stops reaching the model; one agent's never reaches another's | `specs/knowledge/knowledge-lifecycle.spec.ts` | P1 | ✅ |
| Knowledge | An uploaded PDF is indexed by the knowledge worker and answered from | `specs/knowledge/knowledge-pdf.spec.ts` | P2 | ✅ |
| AI config | Rejected key is reported (exact message) and agents keep working; the saved key is the one used | `specs/settings/ai-config.org.spec.ts` | P1 | ✅ |
| Tickets | AI opens a ticket → listed → agent changes status | `specs/tickets/ai-ticket.spec.ts` | P1 | ✅ |
| Tickets | Resolving tells the customer in their chat | `specs/tickets/ai-ticket.spec.ts` | P1 | ✅ |
| Tickets | Confirmation window runs out → closed → customer rates → CSAT kept, and the ask is in the thread (worker, `@redis`) | `specs/tickets/ai-ticket.spec.ts` | P2 | ✅ |
| Lead capture | Visitor shares details → appears as a Lead in People with their name and company | `specs/leads/lead-capture.spec.ts` | P1 | ✅ |
| People | Rename, phone, mark customer; international and unique phones; a WhatsApp contact keeps their number | `specs/leads/people.spec.ts` | P2 | ✅ |
| Team | Admin adds an agent who signs in to the inbox; Agent role sees only Inbox/People/Settings | `specs/team/team.spec.ts` | P1 | ✅ |
| Team | Groups management UI | — | P1 | ⬜ |
| Widget JS API | open/close/isOpen; hidden launcher + page's own trigger button | `specs/widget/js-api.spec.ts` | P1 | ✅ |
| Widget placement | Left side; sidebar mode; dashboard display settings with no code, site config wins; welcome message and colour | `specs/widget/placement.spec.ts` | P1 | ✅ |
| Widget placement | `open({message})`, events | — | P1 | ⬜ |
| Token-auth widget | Server-issued token → chat works; no, tampered, other-widget or expired token is refused | `specs/widget/token-auth.spec.ts` | P1 | ✅ |
| Reply buttons | Telegram, WhatsApp, Messenger, Instagram: AI offers buttons, a tap gets the next answer | `specs/channels/reply-buttons.spec.ts` | P2 | ✅ |
| Reply buttons | Widget quick replies; typed-number matching | — | P2 | ⬜ |
| Help center | Only published articles are public; unpublishing takes one down | `specs/help-center/publishing.org.spec.ts` | P2 | ✅ |
| Help center | Search | — | P2 | ⬜ |
| Workflows | One-message flow greets once and answers once with no AI call; a question branches on the answer; AND across two answers (true, false, an answer that looks like code, a non-number); a missing branch stops instead of taking the other | `specs/workflows/workflows.spec.ts` | P2 | ✅ |
| MCP connectors | An agent answers from a real MCP server's tool; header secrets masked on read, kept on save; a server that is down costs only its own tools, and Test says it could not connect | `specs/mcp/connectors.spec.ts` | P2 | ✅ |
| Channels | Telegram, WhatsApp, Messenger, Instagram, Slack, LINE, SMS (Twilio, Plivo), email: a signed inbound message gets the AI reply on the same channel | `specs/channels/replies.spec.ts` | P2 | ✅ |
| Channels | WhatsApp customer asks for a person → agent takes over in the inbox → reply goes out on WhatsApp | `specs/channels/handover.spec.ts` | P2 | ✅ |
| Channels | Email reply is threaded (Re: subject, In-Reply-To) | `specs/channels/email.spec.ts` | P2 | ✅ |
| Channels | Forged webhooks refused on every channel; echoes, receipts and bot messages never reach the model | `specs/channels/forged-webhooks.spec.ts` | P2 | ✅ |
| Channels | A refused send is marked Not delivered; list and long-title buttons stay within WhatsApp's limits | `specs/channels/delivery.spec.ts` | P2 | ✅ |
| Channels | A provider's retry gets one answer (`@redis`) | `specs/channels/retries.spec.ts` | P2 | ✅ |
| Channels | A WhatsApp number or Messenger Page from the customer's own Meta app (manual credentials + app secret) receives messages at its own webhook address; a wrong secret is refused; the address never takes another app's number | `specs/channels/own-meta-app.spec.ts` | P2 | ✅ |
| Channels | A WhatsApp number whose token Meta refuses is flagged, the Integrations card names it, and reconnecting or the next delivered reply clears it; a message-level refusal does not flag it | `specs/channels/needs-reconnect.spec.ts` | P2 | ✅ |
| Channels | Media messages; disconnect | — | P2 | ⬜ |
| Analytics | A conversation counts for its agent; the dashboard renders | `specs/analytics/analytics.spec.ts` | P2 | ✅ |
| CRM | HubSpot install, manual sync and automatic push (worker) with the lead's details | `specs/crm/hubspot.org.spec.ts` | P2 | ✅ |
| CRM | Pipedrive install (Basic-auth token exchange) and a captured lead pushed as organisation, person, lead and note | `specs/crm/pipedrive.org.spec.ts` | P2 | ✅ |
| Integrations | Jira connect from Settings (real browser OAuth round trip) and the AI filing an issue in the agent's project | `specs/integrations/jira.org.spec.ts` | P2 | ✅ |
| Integrations | Shopify install and link, order status answered from the shop, lookup refused without the customer's email | `specs/integrations/shopify.org.spec.ts` | P2 | ✅ |
| Visual and cross-browser widget | Widget on Firefox, WebKit, Android, iPhone; screenshots light/dark | `--project=widget-*`, `specs/visual` | P2 | ✅ (nightly) |
| Accessibility | axe on key screens (dark and light themes) and the widget: no serious findings anywhere; a per-screen baseline catches regressions | `specs/a11y` | P2 | ✅ |
| Performance | Loader/widget size budgets; reply-time budget | `specs/perf` | P2 | ✅ |
| Security basics | HttpOnly/Secure sign-in cookies; no CORS for other origins | `specs/security` | P2 | ✅ |
| Org website and API access | Changing the org's website allows the new site and revokes the old one; it never gets the sign-in cookies | `specs/security/org-domain-cors.org.spec.ts` | P2 | ✅ |
| Live-chat socket origin | The dashboard's socket signs in with its cookies; an org's website, or a `<script>`-tag (JSONP) handshake, never does (JSONP case `@one-worker`) | `specs/security/agent-socket-origin.spec.ts` | P2 | ✅ |
Enterprise journeys are tracked privately in `enterprise_frontend/e2e/FEATURES.md`.
e2e/a11y-baseline/ai-agents-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/ai-agents.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/ai-config-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/ai-config.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/inbox-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/inbox.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/login.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/people-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/people.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/tickets-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/tickets.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/widget-dark.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/a11y-baseline/widget-light.json
+1
View File
@@ -0,0 +1 @@
[]
e2e/eslint.config.js
+47
View File
@@ -0,0 +1,47 @@
import playwright from 'eslint-plugin-playwright'
import tseslint from 'typescript-eslint'
export default tseslint.config(
{ ignores: ['node_modules/', 'test-results/', 'playwright-report/', '.cache/', '.stack/', '.auth/', 'src/api/schema.d.ts'] },
...tseslint.configs.recommended,
{
files: ['**/*.ts'],
languageOptions: { parserOptions: { projectService: true, tsconfigRootDir: import.meta.dirname } },
rules: {
// A missing await is the classic Playwright flake.
'@typescript-eslint/no-floating-promises': 'error',
},
},
{
...playwright.configs['flat/recommended'],
files: ['specs/**/*.ts', 'src/**/*.ts'],
rules: {
...playwright.configs['flat/recommended'].rules,
'playwright/no-wait-for-timeout': 'error',
'playwright/no-force-option': 'error',
'playwright/prefer-web-first-assertions': 'error',
// Steps live in page objects and components, which run inside tests.
'playwright/no-standalone-expect': 'off',
// Shared helpers that assert inside count as assertions.
'playwright/expect-expect': ['error', { assertFunctionNames: ['expect', 'expectUnchanged', 'expectNoNewA11yViolations'] }],
},
},
{
// Setup is idempotent by design: create the org on an empty DB, sign in otherwise.
files: ['specs/setup/**'],
rules: { 'playwright/no-conditional-in-test': 'off', 'playwright/no-conditional-expect': 'off' },
},
{
// Specs describe behaviour; selectors belong in src/pages and src/components,
// seeding in src/api. See AUTHORING.md.
files: ['specs/**/*.ts'],
ignores: ['specs/setup/**'],
rules: {
'no-restricted-syntax': ['error',
{ selector: "CallExpression[callee.property.name='locator']", message: 'Selectors belong in a page object or component (src/pages, src/components).' },
{ selector: "CallExpression[callee.property.name=/^(fetch|post|put|patch|delete)$/][callee.object.name='request']", message: 'Seed through src/api, not raw requests.' },
],
'no-restricted-imports': ['error', { paths: [{ name: '@playwright/test', message: "Import from '@e2e' so every spec gets the shared fixtures." }] }],
},
},
)
e2e/package-lock.json
Generated
+3118
View File
File diff suppressed because it is too large. Load diff
e2e/package.json
+35
View File
@@ -0,0 +1,35 @@
{
"name": "chattermate-e2e",
"private": true,
"type": "module",
"description": "End-to-end tests for ChatterMate (dashboard, widget, backend) — see AUTHORING.md",
"engines": {
"node": ">=22.18"
},
"scripts": {
"test": "./scripts/stack.sh up && playwright test --project=chromium && playwright test --project=org --workers=1 --pass-with-no-tests",
"test:smoke": "playwright test --project=chromium --grep @smoke",
"test:org": "playwright test --project=org --workers=1 --pass-with-no-tests",
"test:enterprise": "E2E_EDITION=enterprise ./scripts/stack.sh up && E2E_EDITION=enterprise playwright test --project=enterprise --project=chromium && E2E_EDITION=enterprise playwright test --project=enterprise-org --project=org --workers=1 --pass-with-no-tests",
"test:ui": "playwright test --ui",
"report": "playwright show-report",
"stack": "./scripts/stack.sh",
"fake-llm": "node services/fake-llm/server.ts",
"lint": "eslint .",
"lint:enterprise": "cd ../frontend/src/modules/enterprise/e2e && ../../../../../e2e/node_modules/.bin/eslint .",
"api:types": "./scripts/api-types.sh",
"typecheck": "tsc --noEmit"
},
"devDependencies": {
"@axe-core/playwright": "4.13.0",
"@modelcontextprotocol/sdk": "1.30.1",
"@playwright/test": "1.63.0",
"@types/node": "^22.10.0",
"eslint": "^9.14.0",
"eslint-plugin-playwright": "^2.1.0",
"openapi-typescript": "7.13.0",
"typescript": "~5.6.3",
"typescript-eslint": "^8.18.0",
"zod": "4.6.5"
}
}
e2e/playwright.config.ts
+115
View File
@@ -0,0 +1,115 @@
import { existsSync } from 'node:fs'
import { defineConfig, devices, type PlaywrightTestConfig } from '@playwright/test'
import { env } from './src/env'
import { storageStatePath } from './src/org-state'
const isCI = !!process.env.CI
// Reuse a stack already started with `./scripts/stack.sh up` — locally for fast
// reruns, and on CI, which starts it first so every service logs to e2e/.stack/.
const reuse = true
const enterprise = env.edition === 'enterprise'
if (enterprise && !existsSync(env.enterpriseSpecDir)) {
throw new Error(`E2E_EDITION=enterprise but ${env.enterpriseSpecDir} is missing — check out the enterprise_frontend submodule`)
}
const signedInAsAdmin = { ...devices['Desktop Chrome'], storageState: storageStatePath('admin') }
const ORG_WIDE = /\.org\.spec\.ts$/
// Tests that need a capability the stack they run on leaves off: Redis, or a
// single backend worker (socket.io polling isn't sticky across several).
const unsupported = [env.redis ? null : /@redis/, env.multiWorker ? /@one-worker/ : null].filter((tag): tag is RegExp => tag !== null)
// Screenshot baselines are Linux-only (generated in CI); they run in their own project.
const VISUAL = /visual\/.*\.spec\.ts$/
const NOT_DEFAULT = [ORG_WIDE, VISUAL]
/**
* Nightly only (E2E_MORE_BROWSERS=1): the customer-facing widget on the other
* engines and on phones. The dashboard stays Chromium-only.
*/
const otherBrowsers: PlaywrightTestConfig['projects'] = process.env.E2E_MORE_BROWSERS
? [
['widget-firefox', devices['Desktop Firefox']],
['widget-webkit', devices['Desktop Safari']],
['widget-android', devices['Pixel 7']],
['widget-iphone', devices['iPhone 14']],
].map(([name, device]) => ({
name: name as string,
dependencies: ['setup'],
testIgnore: NOT_DEFAULT,
grep: /@widget/,
grepInvert: unsupported,
use: { ...(device as object), storageState: storageStatePath('admin') },
}))
: []
// Both editions share the project names `setup`, `chromium` and `org`, so the same
// commands run either way. Each edition brings its own setup (OSS: first-run form;
// enterprise: signup), and both write the same org contract, so the OSS specs in
// `chromium`/`org` run unchanged against the enterprise stack.
const projects: PlaywrightTestConfig['projects'] = [
enterprise
? { name: 'setup', testDir: env.enterpriseSpecDir, testMatch: /.*\.setup\.ts/ }
: { name: 'setup', testMatch: /.*\.setup\.ts/ },
{ name: 'chromium', dependencies: ['setup'], testIgnore: NOT_DEFAULT, grepInvert: unsupported, use: signedInAsAdmin },
// Specs that change org-wide settings (AI config, business hours, …). Run them in a
// separate invocation with --workers=1 (see package.json "test") so they never race
// the parallel suite, and are not skipped when an unrelated parallel test fails.
{ name: 'org', dependencies: ['setup'], testMatch: ORG_WIDE, use: signedInAsAdmin },
...(enterprise
? [
{ name: 'enterprise', testDir: env.enterpriseSpecDir, dependencies: ['setup'], testIgnore: ORG_WIDE, use: signedInAsAdmin },
{ name: 'enterprise-org', testDir: env.enterpriseSpecDir, dependencies: ['setup'], testMatch: ORG_WIDE, use: signedInAsAdmin },
]
: []),
// Widget screenshots, light and dark. Baselines live next to the specs, Linux only.
{ name: 'visual', dependencies: ['setup'], testMatch: VISUAL, use: signedInAsAdmin },
...otherBrowsers,
]
const service = (name: string, url: string, timeout: number) => ({
name,
command: `./scripts/stack.sh ${name}`,
url,
reuseExistingServer: reuse,
timeout,
})
export default defineConfig({
testDir: './specs',
outputDir: './test-results',
fullyParallel: true,
forbidOnly: isCI,
retries: isCI ? 2 : 0,
workers: isCI ? 2 : undefined,
timeout: 60_000,
expect: { timeout: 10_000 },
reporter: isCI
? [['github'], ['list'], ['html', { open: 'never' }]]
: [['list'], ['html', { open: 'never' }]],
use: {
baseURL: env.frontendUrl,
// Every failure, including the first attempt, keeps its trace.
trace: 'retain-on-failure',
screenshot: 'only-on-failure',
video: 'retain-on-failure',
// The dashboard is a PWA; a service worker caching across tests only adds flake.
serviceWorkers: 'block',
// The widget launcher floats forever; with reduced motion (which the loader honours)
// it holds still, so clicks don't wait on an element that is never "stable".
contextOptions: { reducedMotion: 'reduce' },
// CI browsers run in UTC; pin it so a local run behaves the same.
timezoneId: 'UTC',
},
projects,
webServer: [
service('fake-llm', `${env.fakeLlmUrl}/health`, 30_000),
service('fake-externals', `${env.fakeExternalsUrl}/health`, 30_000),
service('fake-mcp', `${env.fakeMcpUrl}/health`, 30_000),
service('customer-site', `${env.customerSiteUrl}/health`, 30_000),
service('mail', `${env.mailUrl}/livez`, 30_000),
service('backend', `${env.backendUrl}/health`, 180_000),
service('frontend', `${env.frontendUrl}/config.js`, 300_000),
],
})
e2e/scripts/api-fuzz.sh
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Property-based checks of the whole API against its own OpenAPI schema
# (Schemathesis): every read endpoint, with generated ids, query strings and
# bodies, must answer without a server error. Runs against a stack already up
# (scripts/stack.sh up) whose org the setup project has created.
#
# scripts/api-fuzz.sh needs `st` on PATH (pip install schemathesis==4.31.0)
set -euo pipefail
E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
API="http://127.0.0.1:18000"
EMAIL="admin@e2e.example.com" # src/env.ts seedUsers.admin
PASSWORD='E2e-Admin-Passw0rd!'
TOKEN="$(curl -fsS -X POST "$API/api/v1/users/login" --data-urlencode "username=$EMAIL" --data-urlencode "password=$PASSWORD" \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["access_token"])')"
mkdir -p "$E2E_DIR/test-results"
# Reads only for now: writes would also need cleanup between examples. Signing out
# would revoke the token mid-run.
exec st run "$API/openapi.json" \
-H "Authorization: Bearer $TOKEN" \
--checks not_a_server_error \
--include-method GET \
--exclude-path-regex '/users/(logout|me)$' \
-n 5 \
--report junit --report-junit-path "$E2E_DIR/test-results/api-fuzz.xml"
e2e/scripts/api-types.sh
Executable
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
# Regenerates src/api/schema.d.ts — TypeScript types for every backend route and
# payload — from the OSS backend's own OpenAPI schema, so a renamed route or field
# fails `npm run typecheck` here instead of a test at runtime.
#
# scripts/api-types.sh write src/api/schema.d.ts
# scripts/api-types.sh --check fail if it is out of date (CI)
set -euo pipefail
E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO_DIR="$(cd "$E2E_DIR/.." && pwd)"
if [[ -z "${E2E_PYTHON:-}" ]]; then
if [[ -x "$REPO_DIR/backend/venv/bin/python" ]]; then E2E_PYTHON="$REPO_DIR/backend/venv/bin/python"; else E2E_PYTHON="python3"; fi
fi
work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT
# The OSS edition's schema: the public suite never types enterprise routes.
(cd "$REPO_DIR/backend" && ENV_FILE="$E2E_DIR/.env.e2e" NO_ENTERPRISE=1 \
DATABASE_URL="postgresql+psycopg://openapi@127.0.0.1:1/none" \
"$E2E_PYTHON" -c 'import json, sys, app.main; from app.core.application import app; json.dump(app.openapi(), open(sys.argv[1], "w"))' \
"$work/openapi.json" >/dev/null 2>&1)
"$E2E_DIR/node_modules/.bin/openapi-typescript" "$work/openapi.json" -o "$work/schema.d.ts" >/dev/null
if [[ "${1:-}" == "--check" ]]; then
diff -q "$work/schema.d.ts" "$E2E_DIR/src/api/schema.d.ts" >/dev/null \
|| { echo "src/api/schema.d.ts is out of date: run e2e/scripts/api-types.sh and commit it" >&2; exit 1; }
else
cp "$work/schema.d.ts" "$E2E_DIR/src/api/schema.d.ts"
fi
e2e/scripts/stack.sh
Executable
+365
View File
@@ -0,0 +1,365 @@
#!/usr/bin/env bash
# Runs the ChatterMate e2e stack natively (no Docker): a throwaway Postgres DB,
# the backend, a production build of the dashboard and the fake LLM — all on
# 127.0.0.1 ports that never collide with a dev stack on localhost.
#
# stack.sh up start everything in the background, wait until healthy
# stack.sh down stop what `up` started
# stack.sh reset fresh database (restarts the backend)
# stack.sh logs [name] tail backend | frontend | fake-llm logs
# stack.sh tick <worker> one pass of a background worker against the stack's DB:
# crm-sync | knowledge | ticket-lifecycle (tests call this
# instead of waiting on always-on workers' poll intervals)
# stack.sh backend|frontend|fake-llm|fake-externals|fake-mcp|customer-site|mail run one service in the foreground
# (what playwright.config.ts webServer calls)
# stack.sh python ARGS... run python with the stack's backend settings (one-off jobs a test needs)
#
# Knobs (all optional):
# E2E_EDITION=oss|enterprise oss (default) blocks the enterprise submodule even if present;
# enterprise also applies the private suite's .env.e2e overlay
# and seed.sql (frontend/src/modules/enterprise/e2e)
# E2E_PG_URL admin URL of a Postgres with pgvector, no db name
# (default postgresql://$USER@127.0.0.1:5432)
# E2E_PYTHON python with backend deps (default backend/venv/bin/python, else python3)
# E2E_FRONTEND_NODE_MODULES frontend node_modules to build with (default frontend/node_modules)
# E2E_REDIS_URL run with Redis on (webhook dedupe, rate limits, socket.io fan-out),
# as production does. Use a database of its own, e.g. redis://127.0.0.1:6379/5:
# the stack shares its cors:origins cache with anything else on it
# E2E_BACKEND_WORKERS run the backend under gunicorn (--preload, as scripts/start.sh does)
# with this many workers, instead of a single uvicorn process
# FASTEMBED_CACHE_PATH where the embedding model is cached (default e2e/.cache/fastembed;
# fastembed's own default is the OS temp dir, which macOS purges)
set -euo pipefail
E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO_DIR="$(cd "$E2E_DIR/.." && pwd)"
STATE_DIR="$E2E_DIR/.stack"
ENTERPRISE_E2E_DIR="$REPO_DIR/frontend/src/modules/enterprise/e2e"
CACHE_DIR="$E2E_DIR/.cache"
export FASTEMBED_CACHE_PATH="${FASTEMBED_CACHE_PATH:-$CACHE_DIR/fastembed}"
EDITION="${E2E_EDITION:-oss}"
DB_NAME="chattermate_e2e"
TPL_NAME="${DB_NAME}_tpl"
PG_URL="${E2E_PG_URL:-postgresql://${USER}@127.0.0.1:5432}"
PG_URL="${PG_URL%/}"
BACKEND_PORT=18000
FRONTEND_PORT=15173
FAKE_LLM_PORT=14010
FAKE_EXTERNALS_PORT=14020
FAKE_MCP_PORT=14030
CUSTOMER_SITE_PORT=13000
MAIL_SMTP_PORT=11025
MAIL_WEB_PORT=18025
if [[ -z "${E2E_PYTHON:-}" ]]; then
if [[ -x "$REPO_DIR/backend/venv/bin/python" ]]; then E2E_PYTHON="$REPO_DIR/backend/venv/bin/python"; else E2E_PYTHON="python3"; fi
fi
NODE_MODULES="${E2E_FRONTEND_NODE_MODULES:-$REPO_DIR/frontend/node_modules}"
log() { printf '\033[36m[e2e]\033[0m %s\n' "$*" >&2; }
die() { printf '\033[31m[e2e]\033[0m %s\n' "$*" >&2; exit 1; }
case "$EDITION" in oss|enterprise) ;; *) die "E2E_EDITION must be oss or enterprise, got '$EDITION'";; esac
if [[ "$EDITION" == enterprise && ! -d "$ENTERPRISE_E2E_DIR" ]]; then
die "E2E_EDITION=enterprise needs the enterprise_frontend submodule (with its e2e/ folder) checked out"
fi
# The suite drops and recreates this database on every run. Refuse anything that
# is not obviously a throwaway, so a typo can never point it at a dev DB.
[[ "$DB_NAME" == *_e2e ]] || die "refusing to manage database '$DB_NAME': name must end in _e2e"
psql_admin() { psql "$PG_URL/postgres" -v ON_ERROR_STOP=1 -qtA "$@"; }
# SQLAlchemy needs the driver in the scheme; psql accepts both spellings.
db_url() { local base="${PG_URL#postgres://}"; base="${base#postgresql://}"; echo "postgresql+psycopg://$base/$1"; }
backend_env() {
export ENV_FILE="$E2E_DIR/.env.e2e"
export DATABASE_URL; DATABASE_URL="$(db_url "$1")"
if [[ "$EDITION" == oss ]]; then
export NO_ENTERPRISE=1
else
unset NO_ENTERPRISE
# Enterprise-only settings (SMTP to Mailpit, default AI key, …) live with the
# private suite; exported values override the shared .env.e2e.
if [[ -f "$ENTERPRISE_E2E_DIR/.env.e2e" ]]; then set -a; source "$ENTERPRISE_E2E_DIR/.env.e2e"; set +a; fi
fi
if [[ -n "${E2E_REDIS_URL:-}" ]]; then
export REDIS_ENABLED=true REDIS_URL="$E2E_REDIS_URL"
fi
}
# Fingerprint of everything that shapes the schema: migration contents + edition.
# Contents, not names — amending an unmerged migration must rebuild the template.
schema_fingerprint() {
{
echo "$EDITION"
cat "$REPO_DIR"/backend/alembic/versions/*.py
if [[ "$EDITION" == enterprise ]]; then
cat "$REPO_DIR"/backend/app/enterprise/migrations/versions/*.py "$ENTERPRISE_E2E_DIR/seed.sql" 2>/dev/null || true
fi
} | shasum | cut -c1-16
}
# The template DB holds a freshly migrated schema; every run clones it, which takes
# a second instead of re-running every migration.
ensure_template() {
local fp current
fp="$(schema_fingerprint)"
current="$(psql_admin -c "SELECT shobj_description(oid, 'pg_database') FROM pg_database WHERE datname = '$TPL_NAME'")"
if [[ "$current" == "$fp" ]]; then return; fi
log "building template database $TPL_NAME (schema changed or first run)"
psql_admin -c "DROP DATABASE IF EXISTS $TPL_NAME"
psql_admin -c "CREATE DATABASE $TPL_NAME"
psql "$PG_URL/$TPL_NAME" -v ON_ERROR_STOP=1 -q -c "CREATE EXTENSION IF NOT EXISTS vector"
(
backend_env "$TPL_NAME"
cd "$REPO_DIR/backend"
"$E2E_PYTHON" -m alembic upgrade head
if [[ "$EDITION" == enterprise ]]; then
"$E2E_PYTHON" -m alembic -c app/enterprise/migrations/alembic.ini upgrade head
fi
) >&2
if [[ "$EDITION" == enterprise && -f "$ENTERPRISE_E2E_DIR/seed.sql" ]]; then
psql "$PG_URL/$TPL_NAME" -v ON_ERROR_STOP=1 -q -f "$ENTERPRISE_E2E_DIR/seed.sql"
fi
psql_admin -c "COMMENT ON DATABASE $TPL_NAME IS '$fp'"
}
fresh_database() {
ensure_template
psql_admin -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid()" >/dev/null
psql_admin -c "DROP DATABASE IF EXISTS $DB_NAME"
psql_admin -c "CREATE DATABASE $DB_NAME TEMPLATE $TPL_NAME"
log "fresh database $DB_NAME"
}
run_backend() {
fresh_database
backend_env "$DB_NAME"
# The frontend step builds the widget into this dir; the backend only needs it to exist.
export WIDGET_ASSETS_DIR; WIDGET_ASSETS_DIR="$(widget_assets_dir)"
mkdir -p "$WIDGET_ASSETS_DIR"
cd "$REPO_DIR/backend"
if [[ -n "${E2E_BACKEND_WORKERS:-}" ]]; then
# The same server and flags as production (scripts/start.sh). On macOS, a
# preloaded app that forks trips the Objective-C fork-safety abort.
if [[ "$(uname)" == Darwin ]]; then export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES; fi
log "backend ($EDITION, gunicorn × $E2E_BACKEND_WORKERS${E2E_REDIS_URL:+, redis}) on http://127.0.0.1:$BACKEND_PORT"
exec "$E2E_PYTHON" -m gunicorn app.main:app --workers "$E2E_BACKEND_WORKERS" \
--worker-class uvicorn.workers.UvicornWorker --bind "127.0.0.1:$BACKEND_PORT" \
--timeout 120 --keep-alive 5 --access-logfile - --error-logfile - --preload
fi
log "backend ($EDITION${E2E_REDIS_URL:+, redis}) on http://127.0.0.1:$BACKEND_PORT"
exec "$E2E_PYTHON" -m uvicorn app.main:app --host 127.0.0.1 --port "$BACKEND_PORT"
}
# Python with the running stack's backend settings (database, ENV_FILE, edition),
# for one-off jobs a test needs - e.g. one run of the setup-email sender.
run_python() {
backend_env "$DB_NAME"
cd "$REPO_DIR/backend"
exec "$E2E_PYTHON" "$@"
}
# Copies frontend/ into the cache so e2e builds never touch the developer's dist/,
# .env or the committed widget/loader bundles, and so the OSS edition can leave out
# the enterprise module. Prints rsync's change list (empty = nothing changed).
sync_frontend_copy() {
local dest="$1"
[[ -d "$NODE_MODULES" ]] || die "no frontend node_modules at $NODE_MODULES (run npm ci in frontend/, or set E2E_FRONTEND_NODE_MODULES)"
mkdir -p "$dest"
# The loader bundle is generated inside the copy (build_loader), so keep rsync off it.
local excludes=(--exclude node_modules --exclude 'dist*' --exclude '.env*' --exclude e2e --exclude coverage
--exclude public/webclient/chattermate.min.js)
if [[ "$EDITION" == oss ]]; then excludes+=(--exclude src/modules/enterprise); fi
rsync -a --delete -i "${excludes[@]}" "$REPO_DIR/frontend/" "$dest/"
ln -sfn "$NODE_MODULES" "$dest/node_modules"
}
widget_assets_dir() { echo "$CACHE_DIR/widget-assets-$EDITION"; }
# The widget bundle the backend serves at /assets, built from the current source
# (backend/assets holds the committed production build, which may lag the source).
build_widget() {
local src="$1" out
out="$(widget_assets_dir)"
(cd "$src" && node_modules/.bin/vite build --config vite.widget.config.ts --outDir "$out" --emptyOutDir --logLevel warn) >&2
# Same post-steps as scripts/build-widget.ts: the stylesheet can land in unused/,
# and only widget.js + widget.css belong in the assets dir (vite also copies public/).
if [[ ! -f "$out/widget.css" ]]; then cp "$(ls "$out"/unused/*.css | head -n 1)" "$out/widget.css"; fi
find "$out" -mindepth 1 -maxdepth 1 ! -name widget.js ! -name widget.css -exec rm -rf {} +
}
# The embed loader (chattermate.min.js), built from source into the dashboard copy's
# public/ so vite ships it in the build — what build-webclient-*.ts does.
build_loader() {
local src="$1"
(cd "$src" && node_modules/.bin/esbuild src/webclient/chattermate.js --bundle --minify --format=iife \
--target=es2015 --loader:.svg=text --log-level=warning \
--define:process.env.NODE_ENV='"production"' \
--define:__CHATTERMATE_API_URL__="\"http://127.0.0.1:$BACKEND_PORT/api/v1\"" \
--outfile=public/webclient/chattermate.min.js) >&2
}
# One copy of frontend/ feeds all three builds: loader, widget, dashboard.
build_frontend() {
local src="$CACHE_DIR/frontend-$EDITION" out="$CACHE_DIR/dist-$EDITION" changes
changes="$(sync_frontend_copy "$src")"
if [[ -n "$changes" || ! -f "$out/index.html" || ! -f "$(widget_assets_dir)/widget.js" || -n "${E2E_REBUILD_FRONTEND:-}" ]]; then
log "building loader, widget and dashboard ($EDITION)"
build_loader "$src"
build_widget "$src"
(cd "$src" && node_modules/.bin/vite build --outDir "$out" --emptyOutDir --logLevel warn) >&2
fi
# Runtime config: what docker-entrypoint.sh writes in a real deployment.
cat > "$out/config.js" <<EOF
self.APP_CONFIG = {
API_URL: "http://127.0.0.1:$BACKEND_PORT/api/v1",
WS_URL: "ws://127.0.0.1:$BACKEND_PORT",
WIDGET_URL: "http://127.0.0.1:$BACKEND_PORT",
NODE_ENV: "production"
};
EOF
}
run_frontend() {
build_frontend
cd "$CACHE_DIR/frontend-$EDITION"
log "dashboard on http://127.0.0.1:$FRONTEND_PORT"
exec node_modules/.bin/vite preview --outDir "$CACHE_DIR/dist-$EDITION" --host 127.0.0.1 --port "$FRONTEND_PORT" --strictPort
}
run_fake_llm() {
cd "$E2E_DIR"
exec node services/fake-llm/server.ts --port "$FAKE_LLM_PORT"
}
run_fake_externals() {
cd "$E2E_DIR"
exec node services/fake-externals/server.ts --port "$FAKE_EXTERNALS_PORT"
}
run_fake_mcp() {
cd "$E2E_DIR"
exec node services/fake-mcp/server.ts --port "$FAKE_MCP_PORT"
}
run_customer_site() {
cd "$E2E_DIR"
exec node services/customer-site/server.ts --port "$CUSTOMER_SITE_PORT" \
--api "http://127.0.0.1:$BACKEND_PORT/api/v1" \
--loader "http://127.0.0.1:$FRONTEND_PORT/webclient/chattermate.min.js"
}
# SMTP capture (Mailpit): everything the backend emails — email-channel replies,
# account emails, notifications — lands here, readable through its API
# (src/fakes/mailbox.ts).
run_mail() {
command -v mailpit >/dev/null || die "mailpit not found (brew install mailpit, or see https://mailpit.axllent.org)"
exec mailpit --smtp "127.0.0.1:$MAIL_SMTP_PORT" --listen "127.0.0.1:$MAIL_WEB_PORT" --disable-version-check --quiet
}
# One pass of a worker production runs on a loop (docker-compose's knowledge_processor,
# ticket_investigator, crm_sync), with the backend's exact environment.
run_tick() {
local import
case "${1:-}" in
crm-sync) import="from app.workers.crm_sync import run_sync_pass as tick" ;;
knowledge) import="from app.workers.knowledge_processor import run_processor as tick" ;;
# Its main() routes socket emits through Redis and loads extensions first; so must a tick.
ticket-lifecycle) import="from app.core.socketio import configure_socketio; configure_socketio(); from app.channels.app_hooks import load_extensions; load_extensions(); from app.workers.ticket_investigator import run_ticket_lifecycle as tick" ;;
*) die "unknown worker '${1:-}' (crm-sync | knowledge | ticket-lifecycle)" ;;
esac
backend_env "$DB_NAME"
export WIDGET_ASSETS_DIR; WIDGET_ASSETS_DIR="$(widget_assets_dir)"
cd "$REPO_DIR/backend"
"$E2E_PYTHON" -c "import asyncio; $import; asyncio.run(tick())"
}
wait_for() {
local name="$1" url="$2" timeout="$3" i
for ((i = 0; i < timeout; i++)); do
if curl -fsS -o /dev/null "$url" 2>/dev/null; then log "$name is up"; return; fi
if ! kill -0 "$(cat "$STATE_DIR/$name.pid")" 2>/dev/null; then
tail -n 40 "$STATE_DIR/$name.log" >&2; die "$name exited during startup (log: $STATE_DIR/$name.log)"
fi
sleep 1
done
die "$name not healthy after ${timeout}s (log: $STATE_DIR/$name.log)"
}
start_bg() {
local name="$1"
if [[ -f "$STATE_DIR/$name.pid" ]] && kill -0 "$(cat "$STATE_DIR/$name.pid")" 2>/dev/null; then
log "$name already running"; return
fi
"$0" "$name" >"$STATE_DIR/$name.log" 2>&1 &
echo $! >"$STATE_DIR/$name.pid"
}
stop_bg() {
local name="$1" pid
[[ -f "$STATE_DIR/$name.pid" ]] || return 0
pid="$(cat "$STATE_DIR/$name.pid")"
# The service is exec'd, so this pid is the server itself; its children go with it.
pkill -TERM -P "$pid" 2>/dev/null || true
kill -TERM "$pid" 2>/dev/null || true
rm -f "$STATE_DIR/$name.pid"
}
cmd_up() {
mkdir -p "$STATE_DIR"
start_bg fake-llm
wait_for fake-llm "http://127.0.0.1:$FAKE_LLM_PORT/health" 30
start_bg fake-externals
wait_for fake-externals "http://127.0.0.1:$FAKE_EXTERNALS_PORT/health" 30
start_bg fake-mcp
wait_for fake-mcp "http://127.0.0.1:$FAKE_MCP_PORT/health" 30
start_bg customer-site
wait_for customer-site "http://127.0.0.1:$CUSTOMER_SITE_PORT/health" 30
start_bg mail
wait_for mail "http://127.0.0.1:$MAIL_WEB_PORT/livez" 30
start_bg backend
wait_for backend "http://127.0.0.1:$BACKEND_PORT/health" 180
start_bg frontend
wait_for frontend "http://127.0.0.1:$FRONTEND_PORT/config.js" 300
log "stack ready — dashboard http://127.0.0.1:$FRONTEND_PORT"
}
cmd_down() { for s in frontend backend customer-site fake-mcp fake-externals fake-llm mail; do stop_bg "$s"; done; log "stack stopped"; }
wait_port_free() {
local port="$1" i
for ((i = 0; i < 30; i++)); do
lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 || return 0
sleep 1
done
die "port $port still in use after 30s"
}
cmd_reset() {
mkdir -p "$STATE_DIR"
stop_bg backend
wait_port_free "$BACKEND_PORT"
start_bg backend
wait_for backend "http://127.0.0.1:$BACKEND_PORT/health" 180
}
case "${1:-}" in
up) cmd_up ;;
down) cmd_down ;;
reset) cmd_reset ;;
logs) tail -n 100 -f "$STATE_DIR/${2:-backend}.log" ;;
tick) run_tick "${2:-}" ;;
backend) run_backend ;;
frontend) run_frontend ;;
fake-llm) run_fake_llm ;;
fake-externals) run_fake_externals ;;
fake-mcp) run_fake_mcp ;;
customer-site) run_customer_site ;;
mail) run_mail ;;
python) shift; run_python "$@" ;;
*) sed -n '2,21p' "$0"; exit 1 ;;
esac
e2e/scripts/upgrade-check.sh
Executable
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
# Migrations must upgrade the database a deployment already has, not only build
# a fresh one (the e2e template is always built at HEAD). This builds a database
# with the last release's migrations, upgrades it with HEAD's, and checks the
# result is the same schema as a database built fresh at HEAD.
#
# scripts/upgrade-check.sh [base-ref] default: the latest v* tag
#
# Same knobs as stack.sh: E2E_PG_URL, E2E_PYTHON. OSS edition (NO_ENTERPRISE=1).
# E2E_PG_DUMP: a pg_dump matching the server's major version (default: pg_dump).
set -euo pipefail
E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
REPO_DIR="$(cd "$E2E_DIR/.." && pwd)"
PG_URL="${E2E_PG_URL:-postgresql://${USER}@127.0.0.1:5432}"; PG_URL="${PG_URL%/}"
DB_NAME="chattermate_upgrade_e2e"
FRESH_DB="chattermate_fresh_e2e"
BASE_REF="${1:-$(git -C "$REPO_DIR" describe --tags --abbrev=0 --match 'v*')}"
if [[ -z "${E2E_PYTHON:-}" ]]; then
if [[ -x "$REPO_DIR/backend/venv/bin/python" ]]; then E2E_PYTHON="$REPO_DIR/backend/venv/bin/python"; else E2E_PYTHON="python3"; fi
fi
log() { printf '\033[36m[upgrade]\033[0m %s\n' "$*" >&2; }
for db in "$DB_NAME" "$FRESH_DB"; do [[ "$db" == *_e2e ]] || { echo "refusing to manage $db" >&2; exit 1; }; done
psql_admin() { psql "$PG_URL/postgres" -v ON_ERROR_STOP=1 -qtA "$@"; }
base="${PG_URL#postgres://}"; base="${base#postgresql://}"
db_url() { echo "postgresql+psycopg://$base/$1"; }
# Settings come from the e2e env file: HEAD reads it through ENV_FILE, and the old
# release gets a copy as its backend/.env (it predates ENV_FILE).
export ENV_FILE="$E2E_DIR/.env.e2e" NO_ENTERPRISE=1
work="$(mktemp -d)"
cleanup() {
git -C "$REPO_DIR" worktree remove --force "$work/base" >/dev/null 2>&1 || true
rm -rf "$work"
for db in "$DB_NAME" "$FRESH_DB"; do psql_admin -c "DROP DATABASE IF EXISTS $db" >/dev/null 2>&1 || true; done
}
trap cleanup EXIT
for db in "$DB_NAME" "$FRESH_DB"; do
psql_admin -c "DROP DATABASE IF EXISTS $db"
psql_admin -c "CREATE DATABASE $db"
psql "$PG_URL/$db" -v ON_ERROR_STOP=1 -qtA -c "CREATE EXTENSION IF NOT EXISTS vector"
done
log "building the schema at $BASE_REF"
git -C "$REPO_DIR" worktree add --detach "$work/base" "$BASE_REF" >/dev/null 2>&1
cp "$E2E_DIR/.env.e2e" "$work/base/backend/.env"
(cd "$work/base/backend" && DATABASE_URL="$(db_url "$DB_NAME")" "$E2E_PYTHON" -m alembic upgrade head)
log "upgrading to $(git -C "$REPO_DIR" rev-parse --short HEAD), and building the same fresh"
cd "$REPO_DIR/backend"
DATABASE_URL="$(db_url "$DB_NAME")" "$E2E_PYTHON" -m alembic upgrade head
DATABASE_URL="$(db_url "$FRESH_DB")" "$E2E_PYTHON" -m alembic upgrade head
# Schema only, without the noise that legitimately differs (dump header, ownership).
# Written to files first, so a failed dump fails the check instead of diffing nothing.
schema() {
"${E2E_PG_DUMP:-pg_dump}" --schema-only --no-owner --no-privileges --no-comments "$PG_URL/$1" >"$work/$1.raw.sql"
grep -vE '^(--|SET |SELECT pg_catalog\.set_config|\\(un)?restrict )' "$work/$1.raw.sql" | sed '/^$/d' >"$work/$1.sql"
[[ -s "$work/$1.sql" ]] || { echo "empty schema dump for $1" >&2; exit 1; }
}
schema "$FRESH_DB"
schema "$DB_NAME"
if ! diff -u "$work/$FRESH_DB.sql" "$work/$DB_NAME.sql" >"$work/schema.diff"; then
echo "an upgraded database differs from a fresh one at HEAD (fresh → upgraded):" >&2
cat "$work/schema.diff" >&2
exit 1
fi
log "ok: $BASE_REF → HEAD upgrades to exactly the schema a fresh install gets"
e2e/services/customer-site/server.ts
+99
View File
@@ -0,0 +1,99 @@
// A pretend customer website with the ChatterMate widget embedded the way the
// dashboard's install snippet does it. Everything about the embed comes from the
// query string:
//
// http://127.0.0.1:13000/?widget=<id>
// &config=<json> → window.chattermateConfig (launcher, position, trigger, …)
// &token=<jwt> → window.chattermateToken (token-authenticated widgets)
// &trigger=1 → adds a page button with data-chattermate-open
// &snippet=iframe → the dashboard's iframe snippet instead of the script
//
// It has to be a real server on 127.0.0.1: Chrome's Local Network Access check
// blocks a page Playwright fabricates (page.route) from loading the loader and
// calling the API on loopback.
//
// Run: node services/customer-site/server.ts --port 13000 --api <api url> --loader <loader url>
import { createServer } from 'node:http'
function arg(name: string, fallback: string): string {
const i = process.argv.indexOf(`--${name}`)
return i > -1 ? process.argv[i + 1] : fallback
}
const port = Number(arg('port', '13000'))
const apiUrl = arg('api', 'http://127.0.0.1:18000/api/v1')
const loaderUrl = arg('loader', 'http://127.0.0.1:15173/webclient/chattermate.min.js')
interface Embed {
widgetId: string
config: string | null
token: string | null
trigger: boolean
iframe: boolean
}
/** The dashboard's iframe snippet (AgentWidgetTab), classed so the page object finds it. */
function iframeSnippet(widgetId: string): string {
const src = `${apiUrl}/widgets/${encodeURIComponent(widgetId)}/data`
return `<iframe class="chattermate-iframe" src="${src}" width="100%" height="600" frameborder="0" title="AI Assistant" allow="clipboard-write"></iframe>`
}
function page(embed: Embed): string {
if (embed.iframe) {
return `<!doctype html>
<html lang="en">
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Acme Store</title></head>
<body>
<h1>Acme Store</h1>
${iframeSnippet(embed.widgetId)}
</body>
</html>`
}
const globals = [
`window.chattermateId = ${JSON.stringify(embed.widgetId)};`,
`window.chattermateBaseUrl = ${JSON.stringify(apiUrl)};`,
// config arrives as JSON text; parse it here so a malformed value fails the request, not the page.
embed.config ? `window.chattermateConfig = ${JSON.stringify(JSON.parse(embed.config))};` : '',
embed.token ? `window.chattermateToken = ${JSON.stringify(embed.token)};` : '',
].filter(Boolean).join('\n ')
return `<!doctype html>
<html lang="en">
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Acme Store</title></head>
<body>
<h1>Acme Store</h1>
${embed.trigger ? '<button type="button" data-chattermate-open>Chat with us</button>' : ''}
<script>
${globals}
</script>
<script src="${loaderUrl}"></script>
</body>
</html>`
}
createServer((req, res) => {
const url = new URL(req.url ?? '/', `http://127.0.0.1:${port}`)
if (url.pathname === '/health') {
res.writeHead(200, { 'content-type': 'text/plain' }).end('ok')
return
}
if (url.pathname !== '/') {
res.writeHead(404).end()
return
}
const params = url.searchParams
let html: string
try {
html = page({
widgetId: params.get('widget') ?? '',
config: params.get('config'),
token: params.get('token'),
trigger: params.get('trigger') === '1',
iframe: params.get('snippet') === 'iframe',
})
} catch {
res.writeHead(400, { 'content-type': 'text/plain' }).end('config is not valid JSON')
return
}
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }).end(html)
}).listen(port, '127.0.0.1', () => console.log(`[customer-site] http://127.0.0.1:${port}/?widget=<id>`))
e2e/services/fake-externals/limits.ts
+95
View File
@@ -0,0 +1,95 @@
// What each provider rejects, from its own published limits — deliberately NOT
// the backend's constants (backend/app/channels/interactive_limits.py), so a
// backend that drifts past a real limit gets the same 400 it would in production.
//
// Each check returns the provider-style error message, or null when the payload
// would be accepted.
type Body = Record<string, unknown> | null
// Provider payloads are walked loosely; typing every Graph/Bot API shape buys nothing here.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type Loose = any
const len = (value: unknown): number => (typeof value === 'string' ? [...value].length : 0)
const bytes = (value: unknown): number => (typeof value === 'string' ? Buffer.byteLength(value) : 0)
function tooLong(field: string, value: unknown, max: number): string | null {
return len(value) > max ? `${field} is ${len(value)} characters; the limit is ${max}` : null
}
function first(...checks: Array<string | null>): string | null {
return checks.find((c) => c !== null) ?? null
}
// https://developers.facebook.com/docs/whatsapp/cloud-api/messages
function whatsapp(body: Body): string | null {
if (!body || body.messaging_product !== 'whatsapp' || !body.type) return null // receipts, typing
if (body.type === 'text') return tooLong('text.body', (body.text as Body)?.body, 4096)
if (body.type !== 'interactive') return null
const interactive = body.interactive as Record<string, Loose>
const bodyText = tooLong('interactive.body.text', interactive?.body?.text, 1024)
if (interactive?.type === 'button') {
const buttons: Loose[] = interactive.action?.buttons ?? []
if (buttons.length < 1 || buttons.length > 3) return `interactive buttons must number 1-3, got ${buttons.length}`
return first(bodyText, ...buttons.map((b) => first(
tooLong('button title', b.reply?.title, 20),
bytes(b.reply?.id) > 256 ? 'button id is over 256 bytes' : null,
)))
}
if (interactive?.type === 'list') {
const sections: Loose[] = interactive.action?.sections ?? []
const rows = sections.flatMap((s) => s.rows ?? [])
if (sections.length > 10) return `a list allows 10 sections, got ${sections.length}`
if (rows.length < 1 || rows.length > 10) return `a list allows 1-10 rows in total, got ${rows.length}`
return first(bodyText, tooLong('list button', interactive.action?.button, 20), ...rows.map((r) => first(
tooLong('row title', r.title, 24),
tooLong('row description', r.description, 72),
bytes(r.id) > 200 ? 'row id is over 200 bytes' : null,
)))
}
return bodyText
}
// https://developers.facebook.com/docs/messenger-platform/send-messages/quick-replies
function pageSend(textMax: number) {
return (body: Body): string | null => {
const message = body?.message as Record<string, Loose> | undefined
if (!message) return null // sender_action (typing)
const replies: Loose[] = message.quick_replies ?? []
if (replies.length > 13) return `quick_replies allows 13, got ${replies.length}`
return first(tooLong('message.text', message.text, textMax), ...replies.map((q) => tooLong('quick reply title', q.title, 20)))
}
}
// https://core.telegram.org/bots/api#sendmessage
function telegram(action: string, body: Body): string | null {
if (action !== 'sendMessage') return null
const keyboard: Loose[] = (body?.reply_markup as Record<string, Loose> | undefined)?.inline_keyboard?.flat() ?? []
return first(
tooLong('text', body?.text, 4096),
...keyboard.map((b) => (bytes(b.callback_data) > 64 ? 'callback_data is over 64 bytes' : null)),
)
}
// https://developers.line.biz/en/reference/messaging-api/#send-push-message
function line(action: string, body: Body): string | null {
if (action !== 'push') return null
const messages: Loose[] = (body?.messages as Loose[]) ?? []
if (messages.length < 1 || messages.length > 5) return `messages must number 1-5, got ${messages.length}`
return first(...messages.map((m) => tooLong('text', m.text, 5000)))
}
/** The provider's rejection for this call, or null. `provider` is the fake's path prefix. */
export function rejection(provider: string, action: string, method: string, body: Body): string | null {
if (method !== 'POST') return null
switch (provider) {
case 'graph': return action === 'messages' ? (body?.messaging_product ? whatsapp(body) : pageSend(2000)(body)) : null
case 'graph-instagram': return action === 'messages' ? pageSend(1000)(body) : null
case 'telegram': return telegram(action, body)
case 'line': return line(action, body)
case 'twilio': return action === 'Messages.json' ? tooLong('Body', body?.Body, 1600) : null
case 'plivo': return action === 'Message' ? tooLong('text', body?.text, 1600) : null
case 'slack': return action === 'chat.postMessage' || action === 'chat.update' ? tooLong('text', body?.text, 40000) : null
default: return null
}
}
e2e/services/fake-externals/server.ts
+548
View File
@@ -0,0 +1,548 @@
// A recording stand-in for the third-party APIs the backend calls on its own:
// Telegram, Meta Graph (WhatsApp / Messenger / Instagram), Slack, LINE, SMS
// providers, HubSpot, Pipedrive, Jira, Shopify. The backend reaches it through the *_API_BASE overrides in
// .env.e2e (see backend/app/core/external_urls.py); each provider is a path prefix.
//
// Every request is recorded (method, path, query, headers, body, the answer) and
// answered like the provider would:
// - the smallest success the adapter accepts, by default;
// - the provider's own rejection when a payload breaks its published limits
// (limits.ts), so a backend that drifts past a real limit fails here too;
// - a scripted answer — a failure, or a reply for a provider with no built-in
// answers — while one the test registered matches (PUT /__faults).
// Tests then assert on what the backend sent ("the bot replied on Telegram with …")
// and read values the provider would have learned, such as the webhook secret
// Telegram receives in setWebhook.
//
// Where a provider tells the backend who the account is, the answer is derived
// from what the test sent, so every test gets its own account:
// Meta tokens `owns:<id>[,<id>…]:<anything>` own those nodes (pages, numbers, IG
// accounts). Graph refuses such a token a
// node another token owns, and debug_token
// reports its first id as the page
// Meta tokens `…:app=<id>.<secret>:…` were issued by the customer's own Meta
// app: GET /app answers that id and checks
// appsecret_proof against that secret (any
// other token is ChatterMate's app)
// Meta POST /<number> {webhook_configuration} verifies the override callback the way
// Meta does — a GET with hub.verify_token
// that must echo hub.challenge — before
// accepting it
// Meta GET /<node> id = <node>
// LINE GET /info userId derived from the channel access token
// Slack oauth.v2.access team id and bot token derived from the OAuth code
// HubSpot oauth/v1/token an access token derived from the code; introspecting it
// (oauth/v1/access-tokens/<token>) gives a hub id from it
// Pipedrive oauth/token tokens derived from the code, and this fake as the
// company's api_domain; users/me gives a company id
// derived from the token
// Jira authorize consent granted at once: redirects to redirect_uri with a
// fresh code and the state; the token is derived from the
// code and accessible-resources gives a site derived from it
// Shopify orders the customer's order with the number they searched for
// (#1001 if none), shipped with a tracking number
//
// A call's `action` is the provider's method (Telegram, Slack) or else the last
// non-empty path segment (`/Account/<id>/Message/` → `Message`).
//
// Control API (src/fakes/externals.ts):
// GET /__calls?provider=&action=&contains=&since= recorded calls, oldest first;
// contains = substring of path or body
// DELETE /__calls forget every call
// PUT /__faults/:id { provider, action?, contains?, status, body?, times? }
// DELETE /__faults/:id
//
// Run: node services/fake-externals/server.ts --port 14020
import { createHash, createHmac } from 'node:crypto'
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
import { rejection } from './limits.ts'
export interface RecordedCall {
at: string
provider: string
/** Provider-level operation: Telegram/Slack method name, else the last path segment. */
action: string
method: string
path: string
query: Record<string, string>
/** Lower-cased request headers — e.g. authorization, to tell which account sent it. */
headers: Record<string, string>
body: unknown
/** HTTP status the fake answered with. */
status: number
/** What the fake answered — e.g. the message_id Telegram assigned. */
response: unknown
}
export interface Fault {
provider: string
action?: string
/** Only calls whose path or body contains this — scope a fault to one test's account or customer. */
contains?: string
status: number
body?: unknown
/** How many matching calls fail; omitted = until the fault is deleted. */
times?: number
}
const MAX_CALLS = 5000
const calls: RecordedCall[] = []
const faults = new Map<string, Fault>()
/** Every node some `owns:` token has claimed — what other tokens are refused. */
const claimed = new Set<string>()
let counter = 0
class HttpError extends Error {
status: number
constructor(status: number, message: string) {
super(message)
this.status = status
}
}
function send(res: ServerResponse, status: number, body: unknown, headers: Record<string, string> = {}) {
if (status === 204) {
res.writeHead(status, headers)
res.end()
return
}
res.writeHead(status, { 'content-type': 'application/json', ...headers })
res.end(JSON.stringify(body))
}
async function readRaw(req: IncomingMessage): Promise<string> {
const chunks: Buffer[] = []
for await (const chunk of req) chunks.push(chunk as Buffer)
return Buffer.concat(chunks).toString('utf8')
}
function parseBody(raw: string, contentType: string): unknown {
if (!raw) return null
if (contentType.includes('application/x-www-form-urlencoded')) return Object.fromEntries(new URLSearchParams(raw))
try {
return JSON.parse(raw)
} catch {
return raw
}
}
function digest(value: string): string {
return createHash('sha256').update(value).digest('hex')
}
/** A stable numeric id per bot token, so reconnecting the same token finds the same bot. */
function botIdFor(token: string): number {
return parseInt(digest(token).slice(0, 10), 16)
}
function bearer(headers: Record<string, string>): string {
return (headers.authorization ?? '').replace(/^Bearer\s+/i, '')
}
// ---------- per-provider answers ----------
interface Call {
provider: string
/** Where the backend reached this fake, e.g. http://127.0.0.1:14020. */
origin: string
action: string
method: string
url: URL
headers: Record<string, string>
body: Record<string, unknown> | null
}
interface Answer {
status: number
body: unknown
headers?: Record<string, string>
}
const ok = (body: unknown): Answer => ({ status: 200, body })
function telegram(action: string, token: string, body: Record<string, unknown> | null): Answer {
switch (action) {
case 'getMe': {
const id = botIdFor(token)
return ok({ ok: true, result: { id, is_bot: true, first_name: 'E2E Bot', username: `e2e_bot_${id}` } })
}
case 'sendMessage':
return ok({ ok: true, result: { message_id: counter, date: Math.floor(Date.now() / 1000), chat: { id: body?.chat_id }, text: body?.text } })
default:
// setWebhook, deleteWebhook, sendChatAction, answerCallbackQuery, editMessageReplyMarkup, …
return ok({ ok: true, result: true })
}
}
/** The node ids a Meta token owns, or null for a token that makes no claim. */
function ownedNodes(token: string): string[] | null {
const match = /^owns:([^:]+):/.exec(token)
if (!match) return null
const nodes = match[1].split(',')
for (const node of nodes) claimed.add(node)
return nodes
}
function graphError(message: string, code = 100): Answer {
return { status: 400, body: { error: { message, type: 'OAuthException', code, fbtrace_id: 'e2e' } } }
}
/** ChatterMate's own Meta app; its secret mirrors META_APP_SECRET in .env.e2e. */
const CHATTERMATE_APP = { id: 'e2e-chattermate-app', secret: 'e2e-meta-app-secret' }
/** The Meta app that issued a token. */
function appOf(token: string): { id: string; secret: string } {
const match = /:app=([^.:]+)\.([^:]+)/.exec(token)
return match ? { id: match[1], secret: match[2] } : CHATTERMATE_APP
}
/** Meta checks an override callback before taking it: the GET must echo the challenge. */
async function overrideCallback(config: Record<string, unknown>): Promise<Answer> {
const uri = String(config.override_callback_uri ?? '')
if (!uri) return ok({ success: true }) // back to the app's own callback
const challenge = `e2e-challenge-${counter}`
const query = new URLSearchParams({ 'hub.mode': 'subscribe', 'hub.verify_token': String(config.verify_token ?? ''), 'hub.challenge': challenge })
const check = await fetch(`${uri}?${query}`, { signal: AbortSignal.timeout(5_000) }).catch(() => null)
if (!check || (await check.text()) !== challenge) {
return graphError('The callback URL or verify token couldn\'t be validated. Please verify the provided information or try again later.', 2200)
}
return ok({ success: true })
}
async function graph({ action, method, url, headers, body }: Call): Promise<Answer> {
if (action === 'app' && method === 'GET') {
const token = bearer(headers)
const app = appOf(token)
const proof = url.searchParams.get('appsecret_proof')
if (proof !== null && proof !== createHmac('sha256', app.secret).update(token).digest('hex')) {
return graphError('Invalid appsecret_proof provided in the API argument')
}
return ok({ id: app.id, name: `E2E app ${app.id}` })
}
if (action === 'debug_token') {
const inspected = ownedNodes(url.searchParams.get('input_token') ?? '')
return ok({ data: { is_valid: Boolean(inspected), type: 'PAGE', profile_id: inspected?.[0] ?? '' } })
}
// `/<provider>/v21.0/<node>[/<edge>]`. A token can't reach a page, number or IG
// account another token owns; anything unclaimed (a customer's PSID, `me`) is fine.
const owned = ownedNodes(bearer(headers))
const node = url.pathname.split('/').filter(Boolean)[2] ?? ''
if (owned && claimed.has(node) && !owned.includes(node)) {
return graphError(`Unsupported request - object with ID '${node}' does not exist, cannot be loaded due to missing permissions, or does not support this operation`)
}
const webhookConfiguration = body?.webhook_configuration
if (method === 'POST' && webhookConfiguration && typeof webhookConfiguration === 'object') {
return overrideCallback(webhookConfiguration as Record<string, unknown>)
}
if (method !== 'GET') {
return ok({ messages: [{ id: `wamid.e2e.${counter}` }], message_id: `m_e2e_${counter}`, recipient_id: body?.recipient ?? null, success: true })
}
return ok({ id: action, name: `E2E ${action}`, username: `e2e_${action}`, data: [] })
}
function answer(call: Call): Answer | Promise<Answer> {
const { provider, action, headers, body } = call
switch (provider) {
case 'graph':
case 'graph-instagram':
return graph(call)
case 'slack':
if (action === 'oauth.v2.access') {
const code = String(body?.code ?? '')
return ok({ ok: true, access_token: `xoxb-e2e-${digest(code).slice(0, 12)}`, bot_user_id: 'U0E2EBOT', team: { id: `T${digest(code).slice(0, 10).toUpperCase()}`, name: 'E2E Workspace' } })
}
return ok({ ok: true, ts: `${Math.floor(Date.now() / 1000)}.${String(counter).padStart(6, '0')}`, channel: body?.channel ?? 'C0E2E' })
case 'line':
// GET /info is the bot's own profile; everything else (push, webhook endpoint) answers an empty 200.
return ok(action === 'info' ? { userId: `U${digest(bearer(headers)).slice(0, 32)}`, displayName: 'E2E LINE Bot', basicId: '@e2e' } : {})
case 'twilio':
return ok({ sid: `SM${String(counter).padStart(32, '0')}`, status: 'queued' })
case 'plivo':
return ok({ message_uuid: [`e2e-${counter}`], api_id: `e2e-${counter}` })
case 'hubspot':
return hubspot(call)
case 'pipedrive-oauth':
return pipedriveOauth(call)
case 'pipedrive':
return pipedrive(call)
case 'jira-auth':
return jiraAuth(call)
case 'jira':
return jira(call)
case 'shopify':
return shopify(call)
default:
return ok({})
}
}
function hubspot({ method, url, body }: Call): Answer {
const path = url.pathname
if (path.endsWith('/oauth/v1/token')) {
const seed = String(body?.code ?? body?.refresh_token ?? '')
return ok({ access_token: `hs-${digest(seed).slice(0, 16)}`, refresh_token: `hs-refresh-${digest(seed).slice(0, 8)}`, expires_in: 1800, token_type: 'bearer' })
}
const introspected = /\/oauth\/v1\/access-tokens\/([^/]+)$/.exec(path)
if (introspected) return ok({ hub_id: parseInt(digest(introspected[1]).slice(0, 7), 16), hub_domain: 'e2e.hubspot.com', token: introspected[1] })
if (path.endsWith('/contacts/batch/upsert')) return ok({ status: 'COMPLETE', results: [{ id: String(counter), new: true }] })
return ok(method === 'GET' ? { results: [] } : { id: String(counter) })
}
/** Insert-ordered, so the oldest entry goes first once a map outgrows `max`. */
function remember<V>(map: Map<string, V>, key: string, value: V, max = 1000) {
map.set(key, value)
if (map.size > max) map.delete(map.keys().next().value as string)
}
// Pipedrive: OAuth on pipedrive-oauth (oauth.pipedrive.com); the API on the
// company's own host, which the token response names — here, this fake.
/** Consent granted at once: back to the app with a fresh code and the state. */
function consentRedirect(url: URL): Answer {
const redirect = url.searchParams.get('redirect_uri')
if (!redirect) return { status: 400, body: { error: 'invalid_request', error_description: 'redirect_uri is required' } }
const back = new URL(redirect)
back.searchParams.set('code', `code-${counter}`)
back.searchParams.set('state', url.searchParams.get('state') ?? '')
return { status: 302, body: {}, headers: { location: back.toString() } }
}
function pipedriveOauth({ action, url, origin, body }: Call): Answer {
if (action === 'authorize') return consentRedirect(url)
if (action === 'revoke') return ok({})
const seed = String(body?.code ?? body?.refresh_token ?? '')
return ok({
access_token: `pd-${digest(seed).slice(0, 16)}`, refresh_token: `pd-refresh-${digest(seed).slice(0, 8)}`,
token_type: 'Bearer', expires_in: 3600, scope: 'base,contacts:full,leads:full', api_domain: `${origin}/pipedrive`,
})
}
function pipedrive({ method, url, headers, body }: Call): Answer {
const path = url.pathname.replace(/^\/pipedrive/, '')
if (path === '/api/v1/users/me') {
return ok({ success: true, data: { id: 1, company_id: parseInt(digest(bearer(headers)).slice(0, 7), 16), company_name: 'E2E Pipedrive Co' } })
}
// Nothing exists yet: searches find nobody, and every create gets a fresh id.
if (path.endsWith('/search')) return ok({ success: true, data: { items: [] } })
if (path === '/api/v1/leads' && method === 'GET') return ok({ success: true, data: [] })
if (method === 'POST' && path === '/api/v1/leads') return ok({ success: true, data: { id: `00000000-0000-4000-8000-${String(counter).padStart(12, '0')}`, title: body?.title } })
if (method === 'POST' || method === 'PATCH') return ok({ success: true, data: { id: counter, ...(body ?? {}) } })
return { status: 404, body: { success: false, error: `e2e fake has no answer for ${method} ${path}` } }
}
// Jira Cloud: OAuth on jira-auth (auth.atlassian.com), the REST API on jira (api.atlassian.com).
function jiraAuth({ action, url, body }: Call): Answer {
if (action === 'authorize') return consentRedirect(url)
// oauth/token, for both the code grant and a refresh.
const seed = String(body?.code ?? body?.refresh_token ?? '')
return ok({ access_token: `jira-${digest(seed).slice(0, 16)}`, refresh_token: `jira-refresh-${digest(seed).slice(0, 8)}`, token_type: 'Bearer', expires_in: 3600, scope: 'read:jira-work write:jira-work read:jira-user offline_access' })
}
const JIRA_PRIORITIES = ['Highest', 'High', 'Medium', 'Low', 'Lowest'].map((name, i) => ({ id: String(i + 1), name }))
const JIRA_ISSUE_TYPES = [
{ id: '10001', name: 'Task', subtask: false, description: 'A small, distinct piece of work.' },
{ id: '10002', name: 'Sub-task', subtask: true, description: 'A part of a task.' },
]
const jiraIssues = new Map<string, { key: string; fields: Record<string, unknown> }>()
function jiraNotFound(message: string): Answer {
return { status: 404, body: { errorMessages: [message], errors: {} } }
}
function jira({ method, url, headers, body }: Call): Answer {
if (url.pathname.endsWith('/oauth/token/accessible-resources')) {
const site = digest(bearer(headers)).slice(0, 12)
return ok([{ id: `cloud-${site}`, url: `https://e2e-${site}.atlassian.net`, name: `e2e-${site}`, scopes: ['read:jira-work', 'write:jira-work'], avatarUrl: '' }])
}
// /jira/ex/jira/<cloud id>/rest/api/3/<resource>
const resource = /\/ex\/jira\/[^/]+\/rest\/api\/3\/(.+)$/.exec(url.pathname)?.[1] ?? ''
if (resource === 'project') return ok([{ id: '10000', key: 'SUP', name: 'Support' }])
if (resource.startsWith('project/')) return ok({ id: '10000', key: resource.slice('project/'.length), name: 'Support', issueTypes: JIRA_ISSUE_TYPES })
if (resource === 'priority') return ok(JIRA_PRIORITIES)
if (resource === 'issue/createmeta') {
return ok({ projects: [{ key: url.searchParams.get('projectKeys'), issuetypes: [{ id: url.searchParams.get('issuetypeIds'), fields: { summary: {}, description: {}, priority: {} } }] }] })
}
if (resource === 'issue' && method === 'POST') {
const fields = (body?.fields ?? {}) as { project?: { key?: string } }
const key = `${fields.project?.key ?? 'SUP'}-${counter}`
remember(jiraIssues, key, { key, fields: { ...fields, status: { name: 'To Do' } } })
return { status: 201, body: { id: String(counter), key, self: `${url.origin}${url.pathname}/${counter}` } }
}
const issueKey = /^issue\/([^/]+)$/.exec(resource)?.[1]
if (issueKey) {
const issue = jiraIssues.get(issueKey)
if (!issue) return jiraNotFound('Issue does not exist or you do not have permission to see it.')
if (method === 'PUT') {
Object.assign(issue.fields, (body?.fields ?? {}) as Record<string, unknown>)
return { status: 204, body: null }
}
return ok({ id: issueKey, ...issue })
}
return jiraNotFound(`e2e fake has no answer for ${method} ${resource}`)
}
// Shopify Admin API: /shopify/<shop>.myshopify.com/admin/...
interface ShopifyOrder { id: string; number: string; email: string }
const shopifyOrders = new Map<string, ShopifyOrder>()
function money(amount: string) {
return { shopMoney: { amount, currencyCode: 'USD' } }
}
function shopifyOrderNode(order: ShopifyOrder) {
return {
id: `gid://shopify/Order/${order.id}`,
name: `#${order.number}`,
email: order.email,
phone: null,
createdAt: '2026-10-01T10:00:00Z',
processedAt: '2026-10-01T10:00:00Z',
updatedAt: '2026-10-02T10:00:00Z',
cancelledAt: null,
cancelReason: null,
displayFinancialStatus: 'PAID',
displayFulfillmentStatus: 'FULFILLED',
subtotalLineItemsQuantity: 1,
currentSubtotalPriceSet: money('40.00'),
currentTotalPriceSet: money('45.00'),
originalTotalPriceSet: money('45.00'),
customer: { id: `gid://shopify/Customer/${order.id}`, firstName: 'E2E', lastName: 'Customer', email: order.email, phone: null },
shippingAddress: { address1: '1 Test Street', address2: null, city: 'Testville', provinceCode: 'CA', zip: '94000', country: 'United States', phone: null, name: 'E2E Customer' },
fulfillments: [{ status: 'SUCCESS', trackingInfo: [{ company: 'UPS', number: `1ZE2E${order.id}`, url: `https://tracking.e2e.test/1ZE2E${order.id}` }] }],
lineItems: { edges: [{ node: { id: `gid://shopify/LineItem/${order.id}`, title: 'E2E Mug', name: 'E2E Mug', quantity: 1, originalUnitPriceSet: money('40.00'), variant: { id: 'gid://shopify/ProductVariant/1', title: 'Default', sku: 'E2E-MUG', image: null } } }] },
}
}
function shopify({ action, body }: Call): Answer {
if (action === 'access_token') {
const seed = String(body?.subject_token ?? body?.code ?? '')
return ok({ access_token: `shpat_${digest(seed).slice(0, 24)}`, scope: 'read_orders,read_products' })
}
const query = String(body?.query ?? '')
const variables = (body?.variables ?? {}) as Record<string, unknown>
if (query.includes('query SearchOrders')) {
// `email:<email>[ AND name:<number>]`, as the backend builds it.
const search = String(variables.query ?? '')
const email = /email:(\S+)/.exec(search)?.[1] ?? ''
const number = /name:(\S+)/.exec(search)?.[1] ?? '1001'
const order = { id: String(parseInt(digest(`${email}#${number}`).slice(0, 8), 16)), number, email }
remember(shopifyOrders, order.id, order)
return ok({ data: { orders: { edges: email ? [{ node: shopifyOrderNode(order) }] : [], pageInfo: { hasNextPage: false, endCursor: null } } } })
}
if (query.includes('query GetOrder')) {
const order = shopifyOrders.get(String(variables.id ?? '').split('/').at(-1) ?? '')
return ok({ data: { order: order ? shopifyOrderNode(order) : null } })
}
return ok({ errors: [{ message: 'e2e fake has no answer for this query' }] })
}
/** How each provider says "bad request", so the backend's error handling sees the real shape. */
function providerRejection(provider: string, message: string): Answer {
switch (provider) {
case 'graph':
case 'graph-instagram': return graphError(message)
case 'telegram': return { status: 400, body: { ok: false, error_code: 400, description: `Bad Request: ${message}` } }
case 'slack': return ok({ ok: false, error: 'msg_too_long', detail: message })
case 'twilio': return { status: 400, body: { code: 21617, message, status: 400 } }
default: return { status: 400, body: { message, error: message } }
}
}
// ---------- faults ----------
function matchingFault(provider: string, action: string, haystack: string): Fault | undefined {
for (const [id, fault] of faults) {
if (fault.provider !== provider) continue
if (fault.action && fault.action !== action) continue
if (fault.contains && !haystack.includes(fault.contains)) continue
if (fault.times !== undefined) {
fault.times -= 1
if (fault.times <= 0) faults.delete(id)
}
return fault
}
return undefined
}
// ---------- http ----------
async function route(req: IncomingMessage, res: ServerResponse) {
const url = new URL(req.url ?? '/', 'http://fake-externals')
const path = url.pathname
if (path === '/health') return send(res, 200, { status: 'ok', calls: calls.length, faults: faults.size })
if (path === '/__calls' && req.method === 'GET') {
const q = url.searchParams
const since = q.get('since')
const contains = q.get('contains')
return send(res, 200, calls.filter((c) =>
(!q.get('provider') || c.provider === q.get('provider'))
&& (!q.get('action') || c.action === q.get('action'))
&& (!since || c.at >= since)
&& (!contains || `${c.path}\n${JSON.stringify(c.body)}`.includes(contains))))
}
if (path === '/__calls' && req.method === 'DELETE') {
calls.length = 0
return send(res, 200, { ok: true })
}
const faultPath = /^\/__faults\/([\w.-]+)$/.exec(path)
if (faultPath && req.method === 'PUT') {
const fault = parseBody(await readRaw(req), 'application/json') as Fault | null
if (!fault?.provider || !fault.status) throw new HttpError(400, 'a fault needs provider and status')
faults.set(faultPath[1], fault)
return send(res, 200, { ok: true })
}
if (faultPath && req.method === 'DELETE') {
faults.delete(faultPath[1])
return send(res, 200, { ok: true })
}
const [, provider = '', ...rest] = path.split('/')
const raw = await readRaw(req)
const headers = Object.fromEntries(Object.entries(req.headers).map(([k, v]) => [k, Array.isArray(v) ? v.join(', ') : v ?? '']))
const body = parseBody(raw, headers['content-type'] ?? '') as Record<string, unknown> | null
const method = req.method ?? 'GET'
counter += 1
let action = rest.filter(Boolean).at(-1) ?? ''
let token = ''
if (provider === 'telegram') {
// /telegram/bot<token>/<method>
token = (rest[0] ?? '').replace(/^bot/, '')
action = rest[1] ?? ''
}
const fault = matchingFault(provider, action, `${path}\n${raw}`)
const rejected = fault ? null : rejection(provider, action, method, body)
let result: Answer
if (fault) result = { status: fault.status, body: fault.body ?? {} }
else if (rejected) result = providerRejection(provider, rejected)
else if (provider === 'telegram') result = telegram(action, token, body)
else result = await answer({ provider, origin: `http://${headers.host ?? '127.0.0.1'}`, action, method, url, headers, body })
calls.push({
at: new Date().toISOString(),
provider, action, method, path,
query: Object.fromEntries(url.searchParams),
headers, body,
status: result.status,
response: result.body,
})
if (calls.length > MAX_CALLS) calls.splice(0, calls.length - MAX_CALLS)
return send(res, result.status, result.body, result.headers)
}
const portArg = process.argv.indexOf('--port')
const port = Number(portArg > -1 ? process.argv[portArg + 1] : 14020)
createServer((req, res) => {
route(req, res).catch((err: unknown) => {
const status = err instanceof HttpError ? err.status : 500
const message = err instanceof Error ? err.message : String(err)
console.error(`[fake-externals] ${req.method} ${req.url} -> ${status}: ${message}`)
send(res, status, { error: message })
})
}).listen(port, '127.0.0.1', () => console.log(`[fake-externals] listening on http://127.0.0.1:${port}`))
e2e/services/fake-llm/contract_check.py
+60
View File
@@ -0,0 +1,60 @@
"""Checks the fake LLM against the backend's real agno code path.
The e2e suite only works if agno accepts what the fake returns. This runs the same
calls the backend makes — the AI-setup key check and a ChatAgent-style structured
reply — through agno against a running fake, so an agno upgrade that changes the
request or response shape fails here with a clear message instead of as flaky e2e
tests. CI runs it right after the stack starts.
cd backend && PYTHONPATH=. ENV_FILE=../e2e/.env.e2e NO_ENTERPRISE=1 \\
python ../e2e/services/fake-llm/contract_check.py [base_url]
"""
import asyncio
import json
import sys
import urllib.request
from agno.agent import Agent
from app.models.schemas.chat import ChatResponse
from app.utils.agno_utils import create_model, test_model_api_key
BASE_URL = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:14010/v1"
CONTROL = BASE_URL.removesuffix("/v1")
SCENARIO = "contract-check"
def put_scenario() -> None:
spec = {"rules": [
{"match": {"source": "human", "flags": "i"},
"reply": {"message": "Connecting you.", "fields": {"transfer_to_human": True}}},
{"reply": {"message": "Happy to help!"}},
]}
req = urllib.request.Request(f"{CONTROL}/__scenarios/{SCENARIO}", data=json.dumps(spec).encode(),
method="PUT", headers={"content-type": "application/json"})
urllib.request.urlopen(req).read()
async def main() -> int:
put_scenario()
assert await test_model_api_key("e2e-key", "OPENAI_COMPATIBLE", "e2e-fake", base_url=BASE_URL), \
"AI-setup key check failed against the fake"
model = create_model("OPENAI_COMPATIBLE", "e2e-key", "e2e-fake", max_tokens=1000, base_url=BASE_URL)
agent = Agent(model=model, instructions=f"Contract check. [[e2e:{SCENARIO}]]",
response_model=ChatResponse, structured_outputs=True, system_message_role="system")
handover = (await agent.arun("I want a human")).content
assert isinstance(handover, ChatResponse), f"structured reply not parsed: {handover!r}"
assert handover.transfer_to_human is True and handover.message == "Connecting you.", handover
plain = (await agent.arun("hello")).content
assert plain.transfer_to_human is False and plain.message == "Happy to help!", plain
print("fake LLM contract OK")
return 0
if __name__ == "__main__":
sys.exit(asyncio.run(main()))
e2e/services/fake-llm/protocol.ts
+77
View File
@@ -0,0 +1,77 @@
// Wire types shared by the fake LLM server and the test-side client (src/fakes/llm.ts).
// Type-only: the server imports this with `import type`, so Node's type stripping
// needs nothing at runtime.
/** A serialisable RegExp. */
export interface Pattern {
source: string
flags: string
}
/**
* What the model "says" for one matched turn.
*
* - `message` / `fields` fill a structured (json_schema) reply — e.g. ChatAgent's
* ChatResponse. `fields` overlays any schema field: transfer_to_human, create_ticket, …
* - `text` is used for plain-text requests (the transfer agent, the key check).
* Falls back to `message`.
*/
export interface Reply {
message?: string
fields?: Record<string, unknown>
text?: string
/** Token counts to report, for metering. Default 1/1. */
usage?: { prompt_tokens: number; completion_tokens: number }
}
/** How the turn goes wrong instead of (or before) answering. */
export interface Fault {
/** Answer with this HTTP status and an OpenAI-style error instead of a completion. */
status?: number
message?: string
/** Wait this long first — past AGENT_RUN_TIMEOUT to exercise the backend's timeout. */
delayMs?: number
/** Return content that is not the JSON the request's schema asks for. */
malformed?: boolean
}
export interface ToolCall {
name: string
arguments: Record<string, unknown>
}
export interface ScenarioRule {
/** Tested against the last user message. Omitted = matches anything (otherwise). */
match?: Pattern
/** Ask the backend to run a tool first; `reply` is sent after the tool result comes back. */
toolCall?: ToolCall
reply: Reply
fault?: Fault
}
export interface ScenarioSpec {
/**
* Some backend calls never see the agent's instructions (the transfer agent's prompt
* has no `[[e2e:<id>]]` marker), so a scenario can also match on the unique agent name.
*/
agentName?: string
rules: ScenarioRule[]
}
export interface RecordedRequest {
at: string
system: string
lastUser: string
lastRole: string
/** 'json_schema:<name>' | 'json_object' | null */
responseFormat: string | null
tools: string[]
/** Contents of the tool-result messages in this request — e.g. what knowledge search returned. */
toolResults: string[]
/** What the fake answered: 'tool_call:<name>' | 'structured' | 'text' | 'error:<status>' | 'malformed' */
answeredWith: string
/** The model the backend asked for. */
model: string
/** The last 6 characters of the API key it sent — which key the org's config used. */
keySuffix: string
}
e2e/services/fake-llm/server.ts
+285
View File
@@ -0,0 +1,285 @@
// A deterministic stand-in for an OpenAI-compatible model, for e2e tests.
//
// The backend's agents are pointed here (AI config model_type OPENAI_COMPATIBLE,
// base_url http://127.0.0.1:14010/v1). Each test registers a *scenario* — scripted
// replies keyed by a `[[e2e:<id>]]` marker in its agent's instructions or by the
// agent's unique name — so parallel tests in one org never see each other's script.
//
// It speaks only what ChatterMate uses: non-streaming /v1/chat/completions, with or
// without a strict json_schema response_format (ChatAgent's ChatResponse), and tool
// calls. Anything else fails loudly instead of returning something plausible.
//
// Control API (used by src/fakes/llm.ts):
// PUT /__scenarios/:id register/replace a ScenarioSpec
// GET /__scenarios/:id/requests what the backend sent for that scenario
// DELETE /__scenarios forget everything (manual use only: scenario
// ids are unique per test, and a reset would
// wipe any other run sharing this fake)
//
// An API key starting with `e2e-invalid` is rejected with 401, like a real provider.
// A rule can also fail on purpose (fault: status, delay, malformed output).
//
// Run: node services/fake-llm/server.ts --port 14010 (Node >= 22.18 strips the types)
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
import type { Fault, RecordedRequest, Reply, ScenarioRule, ScenarioSpec } from './protocol.ts'
type Json = null | boolean | number | string | Json[] | { [key: string]: Json }
// JSON Schema is walked generically (properties, $ref, anyOf, …); typing every
// keyword buys nothing here.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type JsonSchema = { [key: string]: any }
interface ChatMessage {
role: string
content?: string | Array<{ type: string; text?: string }> | null
tool_calls?: unknown[]
}
interface ChatRequest {
model?: string
messages: ChatMessage[]
stream?: boolean
response_format?: { type: string; json_schema?: { name?: string; schema?: JsonSchema } }
tools?: Array<{ type: string; function: { name: string } }>
}
const MARKER = /\[\[e2e:([\w-]+)\]\]/
const DEFAULT_SCENARIO = '__default'
const DEFAULT_REPLY: Reply = { message: 'Default reply from the e2e fake LLM.', text: 'OK' }
const MAX_RECORDED = 50
// Oldest scenarios are forgotten past this, so a fake that outlives many runs stays small.
const MAX_SCENARIOS = 5000
const scenarios = new Map<string, ScenarioSpec>()
const recorded = new Map<string, RecordedRequest[]>()
// ---------- request helpers ----------
function textOf(content: ChatMessage['content']): string {
if (typeof content === 'string') return content
if (Array.isArray(content)) return content.map((part) => part.text ?? '').join('\n')
return ''
}
function selectScenario(req: ChatRequest): string {
const all = req.messages.map((m) => textOf(m.content)).join('\n')
const marker = MARKER.exec(all)
if (marker && scenarios.has(marker[1])) return marker[1]
for (const [id, spec] of scenarios) {
if (spec.agentName && all.includes(spec.agentName)) return id
}
return DEFAULT_SCENARIO
}
function selectRule(spec: ScenarioSpec | undefined, lastUser: string): ScenarioRule | undefined {
if (!spec) return undefined
return spec.rules.find((rule) => !rule.match || new RegExp(rule.match.source, rule.match.flags).test(lastUser))
}
// ---------- structured output ----------
function resolveRef(schema: JsonSchema, root: JsonSchema): JsonSchema {
if (typeof schema.$ref !== 'string') return schema
const path = schema.$ref.replace(/^#\//, '').split('/')
return path.reduce((node: JsonSchema, key: string) => node?.[key], root) ?? {}
}
function isNullable(schema: JsonSchema): boolean {
if (Array.isArray(schema.type)) return schema.type.includes('null')
const variants = schema.anyOf ?? schema.oneOf
return Array.isArray(variants) && variants.some((v: JsonSchema) => v.type === 'null')
}
/** The minimal value that satisfies `schema` — the base every structured reply is built on. */
function defaultFor(rawSchema: JsonSchema, root: JsonSchema): Json {
const schema = resolveRef(rawSchema, root)
if ('default' in schema) return schema.default as Json
if (isNullable(schema)) return null
if (Array.isArray(schema.enum)) return schema.enum[0] as Json
const variants = schema.anyOf ?? schema.oneOf ?? schema.allOf
if (Array.isArray(variants) && variants.length) return defaultFor(variants[0], root)
const type = Array.isArray(schema.type) ? schema.type[0] : schema.type
switch (type) {
case 'string': return ''
case 'boolean': return false
case 'number':
case 'integer': return 0
case 'array': return []
case 'object':
default: {
const out: { [key: string]: Json } = {}
for (const [key, prop] of Object.entries<JsonSchema>(schema.properties ?? {})) out[key] = defaultFor(prop, root)
return out
}
}
}
function structuredContent(req: ChatRequest, reply: Reply): string {
const schema = req.response_format?.json_schema?.schema
const base = schema ? (defaultFor(schema, schema) as { [key: string]: Json }) : {}
const message = reply.message ?? DEFAULT_REPLY.message
return JSON.stringify({ ...base, message, ...(reply.fields as { [key: string]: Json } | undefined) })
}
// ---------- completion ----------
function completion(req: ChatRequest, message: { content: string | null; tool_calls?: unknown[] }, usage = { prompt_tokens: 1, completion_tokens: 1 }) {
return {
id: `chatcmpl-e2e-${Date.now()}`,
object: 'chat.completion',
created: Math.floor(Date.now() / 1000),
model: req.model ?? 'e2e-fake',
choices: [{
index: 0,
message: { role: 'assistant', ...message },
finish_reason: message.tool_calls ? 'tool_calls' : 'stop',
}],
usage: { ...usage, total_tokens: usage.prompt_tokens + usage.completion_tokens },
}
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
/** A scripted failure: an OpenAI-style error status, or garbage where JSON was asked for. */
function faulted(req: ChatRequest, fault: Fault, record: RecordedRequest) {
if (fault.status) {
record.answeredWith = `error:${fault.status}`
throw new HttpError(fault.status, fault.message ?? 'The e2e fake LLM failed on purpose.')
}
record.answeredWith = 'malformed'
return completion(req, { content: '{"message": "this reply is cut off' })
}
async function answer(req: ChatRequest, authorization: string) {
const scenarioId = selectScenario(req)
const lastUserMsg = [...req.messages].reverse().find((m) => m.role === 'user')
const lastUser = textOf(lastUserMsg?.content)
const lastRole = req.messages.at(-1)?.role ?? ''
const rule = selectRule(scenarios.get(scenarioId), lastUser)
const tools = (req.tools ?? []).map((t) => t.function.name)
const format = req.response_format
const record: RecordedRequest = {
at: new Date().toISOString(),
system: req.messages.filter((m) => m.role === 'system' || m.role === 'developer').map((m) => textOf(m.content)).join('\n'),
lastUser,
lastRole,
responseFormat: format ? (format.type === 'json_schema' ? `json_schema:${format.json_schema?.name ?? ''}` : format.type) : null,
tools,
toolResults: req.messages.filter((m) => m.role === 'tool').map((m) => textOf(m.content)),
answeredWith: '',
model: req.model ?? '',
keySuffix: authorization.replace(/^Bearer\s+/i, '').slice(-6),
}
const list = recorded.get(scenarioId) ?? []
list.push(record)
recorded.set(scenarioId, list.slice(-MAX_RECORDED))
if (rule?.fault?.delayMs) await sleep(rule.fault.delayMs)
if (rule?.fault && (rule.fault.status || rule.fault.malformed)) return faulted(req, rule.fault, record)
const usage = rule?.reply.usage
let body
if (rule?.toolCall && lastRole !== 'tool') {
if (!tools.includes(rule.toolCall.name)) {
throw new HttpError(422, `scenario '${scenarioId}' wants tool '${rule.toolCall.name}' but the request only offers [${tools.join(', ')}]`)
}
record.answeredWith = `tool_call:${rule.toolCall.name}`
body = completion(req, {
content: null,
tool_calls: [{
id: `call_e2e_${Date.now()}`,
type: 'function',
function: { name: rule.toolCall.name, arguments: JSON.stringify(rule.toolCall.arguments) },
}],
})
} else if (format && format.type !== 'text') {
record.answeredWith = 'structured'
body = completion(req, { content: structuredContent(req, rule?.reply ?? DEFAULT_REPLY) }, usage)
} else {
const reply = rule?.reply ?? DEFAULT_REPLY
record.answeredWith = 'text'
body = completion(req, { content: reply.text ?? reply.message ?? DEFAULT_REPLY.text! }, usage)
}
return body
}
// ---------- http ----------
class HttpError extends Error {
status: number
constructor(status: number, message: string) {
super(message)
this.status = status
}
}
async function readJson(req: IncomingMessage): Promise<unknown> {
const chunks: Buffer[] = []
for await (const chunk of req) chunks.push(chunk as Buffer)
const raw = Buffer.concat(chunks).toString('utf8')
if (!raw) return {}
try {
return JSON.parse(raw)
} catch {
throw new HttpError(400, 'body is not JSON')
}
}
function send(res: ServerResponse, status: number, body: unknown) {
res.writeHead(status, { 'content-type': 'application/json' })
res.end(JSON.stringify(body))
}
async function route(req: IncomingMessage, res: ServerResponse) {
const url = new URL(req.url ?? '/', 'http://fake-llm')
const path = url.pathname.replace(/\/+$/, '')
const scenarioPath = /^\/__scenarios\/([\w-]+)(\/requests)?$/.exec(path)
if (req.method === 'GET' && path === '/health') return send(res, 200, { status: 'ok', scenarios: scenarios.size })
if (req.method === 'GET' && path === '/v1/models') return send(res, 200, { object: 'list', data: [{ id: 'e2e-fake', object: 'model' }] })
if (req.method === 'POST' && path === '/v1/chat/completions') {
// Lets a test exercise the "invalid API key" path: the backend live-tests keys.
if (/^Bearer e2e-invalid/.test(req.headers.authorization ?? '')) {
throw new HttpError(401, 'Incorrect API key provided.')
}
const body = (await readJson(req)) as ChatRequest
if (body.stream) throw new HttpError(400, 'the e2e fake LLM does not stream — ChatterMate never asks it to; if it does now, extend services/fake-llm')
if (!Array.isArray(body.messages)) throw new HttpError(400, 'messages[] is required')
return send(res, 200, await answer(body, req.headers.authorization ?? ''))
}
if (scenarioPath && req.method === 'PUT' && !scenarioPath[2]) {
const spec = (await readJson(req)) as ScenarioSpec
if (!Array.isArray(spec.rules)) throw new HttpError(400, 'rules[] is required')
scenarios.set(scenarioPath[1], spec)
recorded.delete(scenarioPath[1])
while (scenarios.size > MAX_SCENARIOS) {
const oldest = scenarios.keys().next().value!
scenarios.delete(oldest)
recorded.delete(oldest)
}
return send(res, 200, { ok: true })
}
if (scenarioPath && req.method === 'GET' && scenarioPath[2]) return send(res, 200, recorded.get(scenarioPath[1]) ?? [])
if (req.method === 'DELETE' && path === '/__scenarios') {
scenarios.clear()
recorded.clear()
return send(res, 200, { ok: true })
}
throw new HttpError(404, `no route for ${req.method} ${url.pathname}`)
}
const portArg = process.argv.indexOf('--port')
const port = Number(portArg > -1 ? process.argv[portArg + 1] : process.env.FAKE_LLM_PORT ?? 14010)
createServer((req, res) => {
route(req, res).catch((err: unknown) => {
const status = err instanceof HttpError ? err.status : 500
const message = err instanceof Error ? err.message : String(err)
console.error(`[fake-llm] ${req.method} ${req.url} -> ${status}: ${message}`)
send(res, status, { error: { message, type: 'e2e_fake_llm_error' } })
})
}).listen(port, '127.0.0.1', () => console.log(`[fake-llm] listening on http://127.0.0.1:${port}`))
e2e/services/fake-mcp/server.ts
+88
View File
@@ -0,0 +1,88 @@
// A real MCP server (streamable HTTP, stateless) for the MCP-connector journeys:
// the backend connects to it exactly as it would to a customer's server, lists
// its tools and calls them when the (fake) model asks.
//
// POST /mcp the MCP endpoint
// GET /__requests the headers each MCP request arrived with, oldest first
// DELETE /__requests forget them
//
// Tools: lookup_order(order_id) → "Order <id>: shipped on 2 May via DHL".
//
// Run: node services/fake-mcp/server.ts --port 14030
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'
import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'
import { z } from 'zod'
interface SeenRequest {
at: string
method: string
authorization: string | null
rpcMethod: string | null
}
const seen: SeenRequest[] = []
function mcpServer(): McpServer {
const server = new McpServer({ name: 'e2e-orders', version: '1.0.0' })
server.registerTool('lookup_order', {
description: 'Look up an order by its id and return its shipping status.',
inputSchema: { order_id: z.string().describe('The order id, e.g. A123') },
}, async ({ order_id }) => ({
content: [{ type: 'text', text: `Order ${order_id}: shipped on 2 May via DHL` }],
}))
return server
}
async function readJson(req: IncomingMessage): Promise<unknown> {
const chunks: Buffer[] = []
for await (const chunk of req) chunks.push(chunk as Buffer)
const raw = Buffer.concat(chunks).toString('utf8')
return raw ? JSON.parse(raw) : undefined
}
function send(res: ServerResponse, status: number, body: unknown) {
res.writeHead(status, { 'content-type': 'application/json' })
res.end(JSON.stringify(body))
}
async function route(req: IncomingMessage, res: ServerResponse) {
const path = new URL(req.url ?? '/', 'http://fake-mcp').pathname
if (path === '/health') return send(res, 200, { status: 'ok' })
if (path === '/__requests' && req.method === 'GET') return send(res, 200, seen)
if (path === '/__requests' && req.method === 'DELETE') {
seen.length = 0
return send(res, 200, { ok: true })
}
if (path !== '/mcp') return send(res, 404, { error: `no route for ${path}` })
if (req.method !== 'POST') return send(res, 405, { jsonrpc: '2.0', error: { code: -32000, message: 'Method not allowed.' }, id: null })
const body = await readJson(req)
seen.push({
at: new Date().toISOString(),
method: req.method,
authorization: req.headers.authorization ?? null,
rpcMethod: (body as { method?: string } | undefined)?.method ?? null,
})
// Stateless: a fresh server and transport per request, as the SDK recommends.
const server = mcpServer()
const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined })
res.on('close', () => {
void transport.close()
void server.close()
})
await server.connect(transport)
await transport.handleRequest(req, res, body)
}
const portArg = process.argv.indexOf('--port')
const port = Number(portArg > -1 ? process.argv[portArg + 1] : 14030)
createServer((req, res) => {
route(req, res).catch((err: unknown) => {
const message = err instanceof Error ? err.message : String(err)
console.error(`[fake-mcp] ${req.method} ${req.url} -> 500: ${message}`)
if (!res.headersSent) send(res, 500, { error: message })
})
}).listen(port, '127.0.0.1', () => console.log(`[fake-mcp] listening on http://127.0.0.1:${port}/mcp`))
e2e/specs/a11y/a11y.spec.ts
+45
View File
@@ -0,0 +1,45 @@
import { test, expect, expectNoNewA11yViolations, InboxPage } from '@e2e'
// Serious and critical axe findings, per screen, held to a baseline
// (a11y-baseline/): today's known issues pass, anything new fails, and the
// baseline shrinks as they are fixed. The widget is checked in both themes.
test.describe('accessibility', () => {
test('sign-in page @a11y', async ({ anonymousPage }) => {
await anonymousPage.goto('/login')
await expect(anonymousPage.getByRole('button', { name: /sign in/i })).toBeVisible()
await expectNoNewA11yViolations(anonymousPage, 'login')
})
// The dashboard is dark unless the user picks light (stored as cm-theme); both are checked.
for (const scheme of ['dark', 'light'] as const) {
const suffix = scheme === 'dark' ? '' : '-light'
test(`dashboard screens, ${scheme} theme @a11y`, async ({ page, agent, agentsPage, aiConfigPage, ticketsPage, peoplePage }) => {
// An agent of its own, so the agents screen is the list (with a card to check), not first-run setup.
expect(agent.id).toBeTruthy()
await page.addInitScript((theme) => localStorage.setItem('cm-theme', theme), scheme)
await new InboxPage(page).goto()
expect(await page.evaluate(() => document.documentElement.dataset.theme)).toBe(scheme)
await expectNoNewA11yViolations(page, `inbox${suffix}`)
await agentsPage.goto()
await expectNoNewA11yViolations(page, `ai-agents${suffix}`)
await aiConfigPage.goto()
await expectNoNewA11yViolations(page, `ai-config${suffix}`)
await ticketsPage.goto()
await expectNoNewA11yViolations(page, `tickets${suffix}`)
await peoplePage.goto()
await expectNoNewA11yViolations(page, `people${suffix}`)
})
}
for (const scheme of ['light', 'dark'] as const) {
test(`widget, ${scheme} theme @a11y @widget`, async ({ widget }) => {
await widget.page.emulateMedia({ colorScheme: scheme })
await widget.embed()
await widget.open()
await expect(widget.messageInput).toBeEditable()
await expectNoNewA11yViolations(widget.page, `widget-${scheme}`)
})
}
})
e2e/specs/agents/agent-editor.spec.ts
+25
View File
@@ -0,0 +1,25 @@
import { test, expect, scenarioMarker, unique, AI_REPLY } from '@e2e'
test.describe('agent editor', () => {
test('an agent created and edited in the dashboard answers with its new instructions @agents', async ({ makeAgent, agentsPage, makeWidget, llm }) => {
// With at least one agent, /ai-agents shows the list rather than onboarding.
await makeAgent()
const scenarioId = unique('scn')
const name = `UI Agent ${scenarioId}`
await agentsPage.goto()
const agentId = await agentsPage.createAgent(name)
await agentsPage.openAgent(agentId)
await agentsPage.setInstructions(`Always sign off as the Acme team. ${scenarioMarker(scenarioId)}`)
await llm.script({ name, scenarioId }, (s) => s.otherwise().reply('Hello from the Acme team!'))
const widget = await makeWidget(agentId)
await widget.embed()
await widget.open()
await widget.send('Hi there')
await expect(widget.lastAgentMessage()).toHaveText('Hello from the Acme team!', AI_REPLY)
const prompts = (await llm.requests({ name, scenarioId })).map((request) => request.system)
expect(prompts.join('\n')).toContain('Always sign off as the Acme team.')
})
})
e2e/specs/analytics/analytics.spec.ts
+25
View File
@@ -0,0 +1,25 @@
import { test, expect, AI_REPLY } from '@e2e'
interface AgentPerformance { bot_agents: Array<{ id: string; total_chats: number }> }
test.describe('analytics', () => {
test('a conversation shows up in the agent\'s numbers, and the dashboard renders without errors @analytics', async ({ agent, widget, llm, adminApi, page }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('Hello')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
const chatsOf = async () => (await adminApi.get<AgentPerformance>('/analytics/agent-performance', { params: { time_range: '24h' } }))
.bot_agents.find((a) => a.id === agent.id)?.total_chats
await expect.poll(chatsOf).toBe(1)
const errors: string[] = []
page.on('pageerror', (error) => errors.push(error.message))
await page.goto('/analytics')
await expect(page.getByRole('main').getByRole('heading', { name: 'Total Conversations' })).toBeVisible()
// The shared test org has hundreds of agents; the dashboard lists the busiest,
// so this one's row is checked through the API above.
expect(errors).toEqual([])
})
})
e2e/specs/auth/login.spec.ts
+38
View File
@@ -0,0 +1,38 @@
import { test, expect, storageStatePath } from '@e2e'
test.describe('signed out', () => {
test.use({ storageState: { cookies: [], origins: [] } })
test('admin signs in and lands on the dashboard @smoke @auth', async ({ loginPage, org, page }) => {
await loginPage.goto()
await loginPage.signIn(org.admin.email, org.admin.password)
await expect(page).toHaveURL(/\/ai-agents/)
})
test('a wrong password keeps you on the sign-in page with an error @auth', async ({ loginPage, org, page }) => {
await loginPage.goto()
await loginPage.signIn(org.admin.email, 'not-the-password')
await expect(loginPage.error).toHaveText('Incorrect email or password')
await expect(page).toHaveURL(/\/login/)
})
test('signing out returns to the sign-in page @auth', async ({ loginPage, dashboard, org, page }) => {
// A fresh session, not the shared admin state — logging out must not affect other tests.
await loginPage.goto()
await loginPage.signIn(org.admin.email, org.admin.password)
await expect(page).toHaveURL(/\/ai-agents/)
await dashboard.logout()
await page.goto('/conversations')
await expect(page).toHaveURL(/\/login/)
})
})
test.describe('as a human agent', () => {
// A path, not a read: spec files load before the setup project writes the state.
test.use({ storageState: storageStatePath('human-agent') })
test('the Agent role cannot open AI settings @auth', async ({ dashboard, page }) => {
await page.goto('/settings/ai-config')
await expect(dashboard.accessDenied).toBeVisible()
})
})
e2e/specs/auth/sessions.spec.ts
+88
View File
@@ -0,0 +1,88 @@
import { test, expect, env, ApiClient, createUser, login, unique, type APIRequestContext } from '@e2e'
// Sessions end on the server, not just in the browser: a copied token stops
// working the moment its session is signed out, replaced or deactivated.
const PASSWORD = 'E2e-Session-Passw0rd!'
interface Device { api: ApiClient; accessToken: string; refreshToken: string }
test.describe('sessions', () => {
let contexts: APIRequestContext[] = []
let newContext: () => Promise<APIRequestContext>
test.beforeEach(({ playwright }) => {
newContext = async () => {
const context = await playwright.request.newContext()
contexts.push(context)
return context
}
})
test.afterEach(async () => {
await Promise.all(contexts.map((c) => c.dispose()))
contexts = []
})
/** Signs `email` in on a browser-like context of its own, as one device would. */
async function signIn(email: string, password = PASSWORD): Promise<Device> {
const context = await newContext()
const { token } = await login(new ApiClient(context), email, password)
const refreshToken = (await context.storageState()).cookies.find((c) => c.name === 'refresh_token')!.value
return { api: new ApiClient(context, token), accessToken: token, refreshToken }
}
/** What a stolen copy of the access token gets: a bare Bearer request, no cookies, no renewal. */
async function withStolen(accessToken: string): Promise<number> {
return (await new ApiClient(await newContext(), accessToken).raw('GET', '/chats/recent')).status()
}
/** What a stolen copy of the refresh cookie gets from /users/refresh. */
async function refreshWithStolen(refreshToken: string): Promise<number> {
const response = await (await newContext()).post(`${env.apiUrl}/users/refresh`, { headers: { Cookie: `refresh_token=${refreshToken}` } })
return response.status()
}
async function newTeammate(adminApi: ApiClient): Promise<{ id: string; email: string }> {
const email = `${unique('session')}@e2e.example.com`
const user = await createUser(adminApi, { name: 'Session Teammate', email, password: PASSWORD, role: 'Agent' })
return { id: user.id, email }
}
test('signing out ends the session: copies of both tokens are refused @auth @security', async ({ adminApi }) => {
const device = await signIn((await newTeammate(adminApi)).email)
expect(await withStolen(device.accessToken)).toBe(200)
await device.api.post('/users/logout')
expect(await withStolen(device.accessToken)).toBe(401)
expect(await refreshWithStolen(device.refreshToken)).toBe(401)
})
test('a refresh token is not accepted as an access token @auth @security', async ({ adminApi }) => {
const device = await signIn((await newTeammate(adminApi)).email)
expect(await withStolen(device.refreshToken)).toBe(401)
})
test('deactivating a teammate cuts them off at once @auth @security', async ({ adminApi }) => {
const teammate = await newTeammate(adminApi)
const device = await signIn(teammate.email)
await adminApi.put(`/users/${teammate.id}`, { data: { is_active: false } })
expect(await withStolen(device.accessToken)).toBe(401)
expect(await refreshWithStolen(device.refreshToken)).toBe(401)
})
test('changing your password signs out your other devices, not this one @auth @security', async ({ adminApi }) => {
const { email } = await newTeammate(adminApi)
const laptop = await signIn(email)
const phone = await signIn(email)
await laptop.api.patch('/users/me', { data: { current_password: PASSWORD, password: 'E2e-Session-Passw0rd-2!' } })
expect(await withStolen(laptop.accessToken)).toBe(200)
expect(await withStolen(phone.accessToken)).toBe(401)
expect(await refreshWithStolen(phone.refreshToken)).toBe(401)
})
})
e2e/specs/channels/delivery.spec.ts
+72
View File
@@ -0,0 +1,72 @@
import { test, expect, channels, FakeExternals, unique, AI_REPLY } from '@e2e'
test.describe('when a channel refuses a message', () => {
test.use({ agentSettings: { transferToHuman: true } })
test('the teammate is told it was not delivered, and the next one goes through @channels @handover', async ({ agent, llm, adminApi, request, inbox }) => {
await llm.script(agent, (s) => s.otherwise().handOver('Let me connect you with a teammate.'))
const customer = unique('Wanda')
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith(customer)
await chat.send('I need a person')
await expect.poll(() => chat.replies(), AI_REPLY).toHaveLength(1)
await inbox.goto()
await inbox.openConversationWith(customer)
await inbox.takeOver()
// WhatsApp refuses free-form messages outside the 24-hour window with error 131047.
const restore = await new FakeExternals(request).fail({
provider: 'graph',
action: 'messages',
contains: chat.waId,
status: 400,
body: { error: { message: 'Re-engagement message', type: 'OAuthException', code: 131047, fbtrace_id: 'e2e' } },
times: 1,
})
await inbox.reply('Are you still there?')
await expect(inbox.agentMessage('Are you still there?')).toContainText('Not delivered')
await restore()
await inbox.reply('Here is your tracking link.')
await expect.poll(() => chat.replies(), AI_REPLY).toContain('Here is your tracking link.')
await expect(inbox.agentMessage('Here is your tracking link.')).not.toContainText('Not delivered')
expect(await chat.replies()).not.toContain('Are you still there?')
})
})
test.describe('choices that do not fit a channel\'s buttons', () => {
test.use({ agentSettings: { channelQuickReplies: true } })
test('five options become a WhatsApp list within its limits, and a pick still gets its answer @channels', async ({ agent, llm, adminApi, request }) => {
const options = ['Track my order', 'Returns', 'Change my delivery address', 'Billing question', 'Talk to someone']
await llm.script(agent, (s) => s
.when(/change my delivery address/i).reply('Sure — what is the new address?')
.otherwise().reply('What can I help you with?', { quick_replies: options }))
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Hi')
// More than three buttons is a list; the fake rejects anything past WhatsApp's real limits.
await expect.poll(() => chat.buttons(), AI_REPLY).toHaveLength(5)
const titles = await chat.buttons()
for (const title of titles) expect([...title].length).toBeLessThanOrEqual(24)
await chat.tap(titles[2])
await expect.poll(async () => (await chat.replies()).at(-1), AI_REPLY).toBe('Sure — what is the new address?')
})
test('a button title longer than WhatsApp allows is shortened, and the tap still works @channels', async ({ agent, llm, adminApi, request }) => {
const long = 'Change my delivery address please'
await llm.script(agent, (s) => s
.when(/change my delivery address/i).reply('Sure — what is the new address?')
.otherwise().reply('What can I help you with?', { quick_replies: [long, 'Returns'] }))
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Hi')
await expect.poll(() => chat.buttons(), AI_REPLY).toHaveLength(2)
const [shortened] = await chat.buttons()
expect([...shortened].length).toBeLessThanOrEqual(24)
await chat.tap(shortened)
await expect.poll(async () => (await chat.replies()).at(-1), AI_REPLY).toBe('Sure — what is the new address?')
})
})
e2e/specs/channels/email.spec.ts
+38
View File
@@ -0,0 +1,38 @@
import { test, expect, channels, expectUnchanged, Mailbox, AI_REPLY } from '@e2e'
test('an email reply stays in the customer\'s thread @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help with that.'))
const chat = (await channels.Email({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Can I change my delivery address?')
const mailbox = new Mailbox(request)
await expect.poll(async () => (await mailbox.messagesTo(chat.address)).length, AI_REPLY).toBe(1)
// One question, one model call, one email — a duplicate would arrive later.
expect(await llm.requests(agent)).toHaveLength(1)
await expectUnchanged('emails to the customer', async () => (await mailbox.messagesTo(chat.address)).length)
const [reply] = await mailbox.messagesTo(chat.address)
expect(reply.subject).toBe(`Re: ${chat.subject}`)
// In-Reply-To is what makes mail clients file the answer under the question.
expect((await mailbox.headers(reply.id))['In-Reply-To']).toEqual([chat.lastMessageId])
})
test('a customer\'s follow-up email continues the same conversation @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s
.when(/tuesday/i).reply('Tuesday works — I have noted it.')
.otherwise().reply('Happy to help with that. Which day suits you?'))
const chat = (await channels.Email({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Can I change my delivery day?')
await expect.poll(() => chat.replies(), AI_REPLY).toHaveLength(1)
await chat.send('Tuesday please')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual([
'Happy to help with that. Which day suits you?',
'Tuesday works — I have noted it.',
])
// One conversation for this sender, holding both exchanges — not two.
const conversations = await adminApi.get<Array<{ message_count: number }>>('/chats/recent', { params: { customer_email: chat.address } })
expect(conversations).toHaveLength(1)
expect(conversations[0].message_count).toBe(4)
})
e2e/specs/channels/forged-webhooks.spec.ts
+100
View File
@@ -0,0 +1,100 @@
import {
test, expect, connectEmail, connectLine, connectMessenger, connectSms, connectWhatsAppNumber, digits,
expectUnchanged, hmac, installSlackWorkspace, metaSignature, metaToken, phoneNumber, routeToAgent, sendWebhook,
slackSignature, unique, whatsappPayload, TelegramBot,
} from '@e2e'
// Anyone can POST to a webhook URL. These are the requests a provider would never
// send: unsigned, wrongly signed, replayed — and events that are signed but must
// never start an AI turn (our own echoes, delivery receipts, other bots).
test.describe('forged webhooks are refused', () => {
test('Meta (WhatsApp, Messenger, Instagram): a wrong or missing signature @channels @security', async ({ adminApi, request }) => {
const { phoneNumberId } = await connectWhatsAppNumber(adminApi)
const json = JSON.stringify(whatsappPayload(phoneNumberId, phoneNumber().slice(1), 'Mallory', { type: 'text', text: { body: 'hi' } }))
const wrongKey = `sha256=${hmac('sha256', 'not-the-app-secret', json).toString('hex')}`
expect((await sendWebhook(request, '/webhooks/meta', { json }, { 'x-hub-signature-256': wrongKey })).status()).toBe(403)
expect((await sendWebhook(request, '/webhooks/meta', { json })).status()).toBe(403)
})
test('Slack: a wrong signature, or a correctly signed request replayed later @channels @security', async ({ request }) => {
const json = JSON.stringify({ type: 'event_callback', team_id: 'T0FORGED', event: { type: 'message', channel: 'D1', user: 'U1', text: 'hi', ts: '1.1' } })
const now = String(Math.floor(Date.now() / 1000))
const tenMinutesAgo = String(Math.floor(Date.now() / 1000) - 600)
const wrong = { 'x-slack-request-timestamp': now, 'x-slack-signature': 'v0=00' }
expect((await sendWebhook(request, '/webhooks/slack', { json }, wrong)).status()).toBe(403)
const replayed = { 'x-slack-request-timestamp': tenMinutesAgo, 'x-slack-signature': slackSignature(json, tenMinutesAgo) }
expect((await sendWebhook(request, '/webhooks/slack', { json }, replayed)).status()).toBe(403)
})
test('Telegram: a wrong webhook secret @channels @security', async ({ adminApi, request, agent }) => {
const { account } = await TelegramBot.connect(adminApi, request, agent.id)
const response = await sendWebhook(request, `/webhooks/telegram/${account.id}`, {
json: JSON.stringify({ update_id: 1, message: { message_id: 1, date: 0, chat: { id: 1, type: 'private' }, from: { id: 1, is_bot: false, first_name: 'M' }, text: 'hi' } }),
}, { 'x-telegram-bot-api-secret-token': 'guessed' })
expect(response.status()).toBe(403)
})
test('LINE: a signature made with the wrong channel secret @channels @security', async ({ adminApi, request }) => {
const account = await connectLine(adminApi, unique('line-secret'), unique('line-token'))
const json = JSON.stringify({ destination: account.external_account_id, events: [{ type: 'message', source: { type: 'user', userId: `U${digits(32)}` }, message: { type: 'text', id: '1', text: 'hi' } }] })
const response = await sendWebhook(request, `/webhooks/line/${account.id}`, { json }, {
'x-line-signature': hmac('sha256', 'not-the-channel-secret', json).toString('base64'),
})
expect(response.status()).toBe(403)
})
test('SMS: Twilio and Plivo signatures made with the wrong auth token @channels @security', async ({ adminApi, request }) => {
const number = phoneNumber()
const twilio = await connectSms(adminApi, 'twilio', number, { account_sid: unique('AC'), auth_token: unique('token') })
const form = { From: phoneNumber(), To: number, Body: 'hi', MessageSid: unique('SM') }
const forgedTwilio = await sendWebhook(request, `/webhooks/sms/twilio/${twilio.id}`, { form }, { 'x-twilio-signature': hmac('sha1', 'guessed', 'x').toString('base64') })
expect(forgedTwilio.status()).toBe(403)
const plivo = await connectSms(adminApi, 'plivo', phoneNumber(), { auth_id: unique('MA'), auth_token: unique('token') })
const forgedPlivo = await sendWebhook(request, `/webhooks/sms/plivo/${plivo.id}`, { form: { From: '447700900000', To: '447700900001', Text: 'hi', MessageUUID: unique('u') } }, {
'x-plivo-signature-v3': hmac('sha256', 'guessed', 'x').toString('base64'), 'x-plivo-signature-v3-nonce': 'n',
})
expect(forgedPlivo.status()).toBe(403)
})
test('email: an inbound-parse post without the inbox\'s token @channels @security', async ({ adminApi, request }) => {
const inbox = await connectEmail(adminApi, `${unique('support')}@e2e.example.com`)
const json = JSON.stringify({ from: 'mallory@example.com', to: inbox.external_account_id, subject: 'hi', text: 'hi' })
expect((await sendWebhook(request, `/webhooks/email/${inbox.id}?token=guessed`, { json })).status()).toBe(403)
expect((await sendWebhook(request, `/webhooks/email/${inbox.id}`, { json })).status()).toBe(403)
})
})
test.describe('signed events that must not start an AI turn', () => {
test('a Messenger echo of our own message, a WhatsApp delivery receipt and another bot\'s Slack message @channels', async ({ adminApi, request, agent, llm }) => {
await llm.script(agent, (s) => s.otherwise().reply('This should never be sent.'))
// Messenger: Meta echoes the page's own sends back with is_echo.
const pageId = digits(15)
const page = await connectMessenger(adminApi, pageId, metaToken(pageId))
await routeToAgent(adminApi, page.id, agent.id)
const echo = { object: 'page', entry: [{ id: pageId, messaging: [{ sender: { id: pageId }, recipient: { id: digits(16) }, timestamp: Date.now(), message: { mid: unique('m'), text: 'Hello from us', is_echo: true } }] }] }
expect((await sendWebhook(request, '/webhooks/meta', { json: JSON.stringify(echo) }, metaSignature(JSON.stringify(echo)))).ok()).toBe(true)
// WhatsApp: delivery/read receipts arrive as `statuses`, with no messages.
const number = await connectWhatsAppNumber(adminApi)
await routeToAgent(adminApi, number.account.id, agent.id)
const receipt = { object: 'whatsapp_business_account', entry: [{ id: '1', changes: [{ field: 'messages', value: { messaging_product: 'whatsapp', metadata: { phone_number_id: number.phoneNumberId }, statuses: [{ id: 'wamid.x', status: 'read', timestamp: '1', recipient_id: '447700900000' }] } }] }] }
expect((await sendWebhook(request, '/webhooks/meta', { json: JSON.stringify(receipt) }, metaSignature(JSON.stringify(receipt)))).ok()).toBe(true)
// Slack: messages posted by a bot (including ours) carry bot_id.
const workspace = await installSlackWorkspace(adminApi, request)
await routeToAgent(adminApi, workspace.account.id, agent.id)
const teamId = workspace.teamId
const now = String(Math.floor(Date.now() / 1000))
const botMessage = JSON.stringify({ type: 'event_callback', team_id: teamId, event_id: unique('Ev'), event: { type: 'message', channel_type: 'im', channel: 'D1', bot_id: 'B1', text: 'beep', ts: `${now}.000100` } })
expect((await sendWebhook(request, '/webhooks/slack', { json: botMessage }, { 'x-slack-request-timestamp': now, 'x-slack-signature': slackSignature(botMessage, now) })).ok()).toBe(true)
// Each was accepted (a provider would retry a refusal) and none reached the model.
await expectUnchanged('model calls', async () => (await llm.requests(agent)).length, 5_000)
expect(await llm.requests(agent)).toHaveLength(0)
})
})
e2e/specs/channels/handover.spec.ts
+37
View File
@@ -0,0 +1,37 @@
import { test, expect, channels, expectUnchanged, unique, AI_REPLY } from '@e2e'
// The inbox side is the same for every channel, and a human's reply leaves through
// the same adapter call as the AI's (covered per channel in replies.spec), so one
// channel is enough here.
test.describe('human handover on a channel', () => {
test.use({ agentSettings: { transferToHuman: true } })
test('a WhatsApp customer asks for a person, the teammate\'s reply reaches them on WhatsApp, and the AI stays out of it @channels @handover', async ({ agent, llm, adminApi, request, inbox }) => {
await llm.script(agent, (s) => s
.when(/human|person/i).handOver('Let me connect you with a teammate.')
.otherwise().reply('Happy to help!'))
const customer = unique('Wanda')
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith(customer)
await chat.send('I want to talk to a person')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['Let me connect you with a teammate.'])
await inbox.goto()
await inbox.openConversationWith(customer)
await expect(inbox.lastCustomerMessage()).toContainText('I want to talk to a person')
await inbox.takeOver()
await inbox.reply('Hi, this is Sam. How can I help?')
// The takeover itself tells the customer a person has joined.
await expect.poll(() => chat.replies(), AI_REPLY).toEqual([
'Let me connect you with a teammate.',
'You\'re now connected with a member of our team.',
'Hi, this is Sam. How can I help?',
])
const modelCalls = (await llm.requests(agent)).length
await chat.send('My parcel never arrived')
await expect(inbox.lastCustomerMessage()).toContainText('My parcel never arrived')
await expectUnchanged('WhatsApp replies after takeover', () => chat.replies())
expect((await llm.requests(agent))).toHaveLength(modelCalls)
})
})
e2e/specs/channels/needs-reconnect.spec.ts
+94
View File
@@ -0,0 +1,94 @@
import {
test, expect, connectWhatsAppNumber, expectUnchanged, listChannelAccounts, routeToAgent,
ApiClient, FakeExternals, IntegrationsPage, WhatsAppChat, AI_REPLY,
} from '@e2e'
// When Meta stops accepting a number's access token (an expired temporary
// token, a revoked system user), every reply fails until someone reconnects
// the number. The account is flagged so the Integrations card can say so,
// rather than replies failing with nothing on screen.
//
// Other tests share the org and its WhatsApp card, so the card is checked for
// this test's number by name, never for the absence of any warning at all.
const TOKEN_REFUSED = { error: { message: 'Error validating access token: Session has expired', type: 'OAuthException', code: 190, fbtrace_id: 'e2e' } }
// Each read lists every account in the shared org; once a second is plenty
const FLAG_POLL = { ...AI_REPLY, intervals: [1_000] }
const WINDOW_CLOSED = { error: { message: 'Re-engagement message', type: 'OAuthException', code: 131047, fbtrace_id: 'e2e' } }
async function needsReconnect(api: ApiClient, accountId: string): Promise<boolean> {
const account = (await listChannelAccounts(api)).find((a) => a.id === accountId)
return account?.needs_reconnect ?? false
}
test.describe('a WhatsApp number whose token Meta refuses', () => {
test('is flagged, the Integrations card names it, and reconnecting clears it @channels', async ({ agent, llm, adminApi, request, page }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const number = await connectWhatsAppNumber(adminApi)
await routeToAgent(adminApi, number.account.id, agent.id)
const externals = new FakeExternals(request)
const chat = new WhatsAppChat(request, externals, number, 'Tess')
const restore = await externals.fail({ provider: 'graph', action: 'messages', contains: chat.waId, status: 401, body: TOKEN_REFUSED })
await chat.send('Hello?')
await expect.poll(() => needsReconnect(adminApi, number.account.id), FLAG_POLL).toBe(true)
expect(await chat.replies()).toEqual([])
const integrations = new IntegrationsPage(page)
const warning = integrations.expiredAccessFor(number.account.display_name ?? '')
await integrations.goto()
await expect(integrations.card('WhatsApp')).toContainText(warning)
// Other numbers are connected too; Manage still leads to the fix, not the first number
await integrations.manage('WhatsApp')
await expect(integrations.reconnectPrompt()).toBeVisible()
// The customer swaps in a working token: same number, connected again
await restore()
const reconnected = await connectWhatsAppNumber(adminApi, number.phoneNumberId)
expect(reconnected.account.id).toBe(number.account.id)
expect(reconnected.account.needs_reconnect).toBe(false)
await integrations.goto()
await expect(integrations.card('WhatsApp')).not.toContainText(warning)
// Replies now go out with the new token
const afterReconnect = new WhatsAppChat(request, externals, reconnected, 'Tess')
await afterReconnect.send('Are you there now?')
await expect.poll(() => afterReconnect.replies(), AI_REPLY).toEqual(['Happy to help!'])
})
test('is healthy again as soon as a reply goes through @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const number = await connectWhatsAppNumber(adminApi)
await routeToAgent(adminApi, number.account.id, agent.id)
const externals = new FakeExternals(request)
const chat = new WhatsAppChat(request, externals, number, 'Tess')
// Meta refuses the token once, then accepts it (it was refreshed on Meta's side)
await externals.fail({ provider: 'graph', action: 'messages', contains: chat.waId, status: 401, body: TOKEN_REFUSED, times: 1 })
await chat.send('Hello?')
await expect.poll(() => needsReconnect(adminApi, number.account.id), FLAG_POLL).toBe(true)
await chat.send('Hello again?')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['Happy to help!'])
expect(await needsReconnect(adminApi, number.account.id)).toBe(false)
})
test('is not flagged when Meta refuses one message for its own reason @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const number = await connectWhatsAppNumber(adminApi)
await routeToAgent(adminApi, number.account.id, agent.id)
const externals = new FakeExternals(request)
const chat = new WhatsAppChat(request, externals, number, 'Tess')
// A closed 24-hour window refuses the message, not the token
const restore = await externals.fail({ provider: 'graph', action: 'messages', contains: chat.waId, status: 400, body: WINDOW_CLOSED })
await chat.send('Hello?')
const refused = async () => (await externals.calls('graph', 'messages', { contains: chat.waId }))
.filter((c) => c.status === 400).length
await expect.poll(refused, AI_REPLY).toBeGreaterThan(0)
await expectUnchanged('the number\'s reconnect flag', () => needsReconnect(adminApi, number.account.id))
expect(await needsReconnect(adminApi, number.account.id)).toBe(false)
await restore()
})
})
e2e/specs/channels/own-meta-app.spec.ts
+80
View File
@@ -0,0 +1,80 @@
import {
test, expect, connectMessenger, connectWhatsAppNumber, digits, expectUnchanged, metaVerifiesCallback, ownMetaApp,
pageChat, pastedCallback, routeToAgent, whereMetaSends, FakeExternals, WhatsAppChat, AI_REPLY,
} from '@e2e'
// A WhatsApp number or Facebook Page connected with credentials from the
// customer's OWN Meta app — the only way to try Meta's free test number. Meta
// signs that app's webhooks with its own secret, so each such account has its
// own webhook address, checked with that secret only (issue #353).
//
// Connected through the API: the org is shared, and once any test has a number
// the Integrations card offers Manage, not Connect. The card itself is covered
// by the dashboard's unit tests.
test.describe('a number or Page from the customer\'s own Meta app', () => {
test('a WhatsApp number receives messages: its webhooks are pointed at ChatterMate on connect @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s
.when(/opening hours/i).reply('We are open 9am to 5pm, Monday to Friday.')
.otherwise().reply('Happy to help!'))
const app = ownMetaApp()
const number = await connectWhatsAppNumber(adminApi, digits(15), app)
// Meta took the per-number callback, after checking it answers: nothing to paste
expect(number.account.webhook_auto_configured).toBe(true)
await routeToAgent(adminApi, number.account.id, agent.id)
const externals = new FakeExternals(request)
const chat = new WhatsAppChat(request, externals, number, 'Tess', () => whereMetaSends(externals, number.phoneNumberId, app))
await chat.send('What are your opening hours?')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['We are open 9am to 5pm, Monday to Friday.'])
})
test('a Messenger Page receives messages once its callback URL and verify token are pasted into the app @channels', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Hi! How can we help?'))
const app = ownMetaApp()
const pageId = digits(15)
const token = app.token(pageId)
const account = await connectMessenger(adminApi, pageId, token, app.secret)
await routeToAgent(adminApi, account.id, agent.id)
const url = account.webhook_url ?? ''
const verifyToken = account.webhook_verify_token ?? ''
// The customer pastes both into their app; Meta checks the address before saving
expect(await metaVerifiesCallback(request, url, verifyToken)).toBe(true)
expect(await metaVerifiesCallback(request, url, 'a-guessed-token')).toBe(false)
const chat = pageChat('Messenger', request, new FakeExternals(request), 'page', pageId, token, 'Remi', pastedCallback(url, app))
await chat.send('Hello there')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['Hi! How can we help?'])
})
test('a wrong app secret is refused when connecting @channels', async ({ adminApi }) => {
const app = ownMetaApp()
await expect(connectWhatsAppNumber(adminApi, digits(15), { ...app, secret: 'not-the-app-secret' }))
.rejects.toThrow(/400: .*That app secret is not the one for the Meta app this token comes from/)
})
test('its address takes only its own app\'s accounts, never another app\'s number @channels @security', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('This should never be sent.'))
const mine = ownMetaApp()
const ourNumber = await connectWhatsAppNumber(adminApi, digits(15), mine)
const theirNumber = await connectWhatsAppNumber(adminApi, digits(15), ownMetaApp())
await routeToAgent(adminApi, theirNumber.account.id, agent.id)
const externals = new FakeExternals(request)
// Signed with our app's secret and sent to our address, but naming their number
const forged = new WhatsAppChat(request, externals, theirNumber, 'Mallory', () => whereMetaSends(externals, ourNumber.phoneNumberId, mine))
await forged.send('Hi, I am not who I say I am')
await expectUnchanged('replies on the other app\'s number', () => forged.replies())
})
test('without the app secret a Page has no address of its own @channels', async ({ adminApi }) => {
const pageId = digits(15)
const account = await connectMessenger(adminApi, pageId, ownMetaApp().token(pageId))
expect(account.webhook_url).toBeFalsy()
expect(account.webhook_verify_token).toBeFalsy()
})
})
e2e/specs/channels/replies.spec.ts
+19
View File
@@ -0,0 +1,19 @@
import { test, expect, channels, expectUnchanged, AI_REPLY } from '@e2e'
// The core promise of every channel: a customer writes in, the agent's answer goes
// back out on the same channel, to that customer.
for (const [name, connect] of Object.entries(channels)) {
test(`a customer on ${name} gets the agent's answer back there @channels`, async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s
.when(/opening hours/i).reply('We are open 9am to 5pm, Monday to Friday.')
.otherwise().reply('Happy to help!'))
const channel = await connect({ api: adminApi, request, agentId: agent.id })
const chat = channel.chatWith('Tess')
await chat.send('What are your opening hours?')
// Exactly one answer, the scripted one, from this account — no duplicate, no default.
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['We are open 9am to 5pm, Monday to Friday.'])
await expectUnchanged(`${name} replies`, () => chat.replies())
})
}
e2e/specs/channels/reply-buttons.spec.ts
+21
View File
@@ -0,0 +1,21 @@
import { test, expect, channelsWithButtons, AI_REPLY } from '@e2e'
test.describe('reply buttons', () => {
test.use({ agentSettings: { channelQuickReplies: true } })
for (const [name, connect] of Object.entries(channelsWithButtons)) {
test(`the AI offers buttons on ${name} and a tap gets the next answer @channels`, async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s
.when(/track my order/i).reply('Your order ships tomorrow.')
.otherwise().reply('What can I help you with?', { quick_replies: ['Track my order', 'Returns'] }))
const chat = (await connect({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Hi')
await expect.poll(() => chat.buttons(), AI_REPLY).toEqual(['Track my order', 'Returns'])
await chat.tap('Track my order')
await expect.poll(async () => (await chat.replies()).at(-1), AI_REPLY).toContain('Your order ships tomorrow.')
expect(await chat.replies()).toHaveLength(2)
})
}
})
e2e/specs/channels/retries.spec.ts
+23
View File
@@ -0,0 +1,23 @@
import { test, expect, connectWhatsAppNumber, deliverMeta, expectUnchanged, phoneNumber, routeToAgent, whatsappPayload, FakeExternals, AI_REPLY } from '@e2e'
// Providers retry a webhook they think failed. Dedupe lives in Redis, which only
// the nightly Redis run turns on (scripts/stack.sh E2E_REDIS_URL); playwright.config.ts
// leaves @redis tests out otherwise.
test('a provider delivering the same message twice gets one answer @channels @redis', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Answered once.'))
const number = await connectWhatsAppNumber(adminApi)
await routeToAgent(adminApi, number.account.id, agent.id)
const waId = phoneNumber().slice(1)
// The exact same delivery, message id included, twice.
const payload = whatsappPayload(number.phoneNumberId, waId, 'Rita', { type: 'text', text: { body: 'Hello?' } })
await deliverMeta(request, payload)
await deliverMeta(request, payload)
const sent = async () => (await new FakeExternals(request).calls('graph', 'messages', { contains: waId }))
.filter((c) => (c.body as { type?: string }).type === 'text').length
await expect.poll(sent, AI_REPLY).toBe(1)
await expectUnchanged('replies to the retried message', sent)
expect(await llm.requests(agent)).toHaveLength(1)
})
e2e/specs/crm/hubspot.org.spec.ts
+63
View File
@@ -0,0 +1,63 @@
import { test, expect, connectCrm, disconnectCrm, findPerson, FakeExternals, runWorkerOnce, syncPersonToCrm, unique, AI_REPLY } from '@e2e'
// One HubSpot connection per org, so this runs in the serial `org` project and
// disconnects afterwards.
test.describe('HubSpot', () => {
test.use({ agentSettings: { leadCapture: true } })
test.afterEach(async ({ adminApi }) => { await disconnectCrm(adminApi, 'hubspot') })
test('a captured lead is pushed to HubSpot with what they told us @crm', async ({ agent, widget, llm, adminApi, request }) => {
const code = unique('hs-code')
await connectCrm(adminApi, 'hubspot', code)
const { email, name } = widget.customer
await llm.script(agent, (s) => s
.when(/@/).captureLead('Thanks! Someone from sales will reach out.', { email, name, company: 'Acme Corp' })
.otherwise().reply('Happy to set that up — what is your email?'))
await widget.embed()
await widget.open()
await widget.send('I would like a demo')
await expect(widget.lastAgentMessage()).toHaveText('Happy to set that up — what is your email?', AI_REPLY)
await widget.send(`Sure, I am ${name}, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Thanks! Someone from sales will reach out.', AI_REPLY)
const person = await findPerson(adminApi, email)
const status = await syncPersonToCrm(adminApi, person.id)
expect(status.synced.map((s) => s.provider)).toContain('hubspot')
// What HubSpot received: the contact keyed by email, as a lead, with their company,
// sent with the token this org's install was granted.
const externals = new FakeExternals(request)
const [upsert] = await externals.calls('hubspot', 'upsert', { contains: email })
const [grant] = await externals.calls('hubspot', 'token', { contains: code })
expect(upsert.headers.authorization).toBe(`Bearer ${(grant.response as { access_token: string }).access_token}`)
expect(upsert.body).toMatchObject({
inputs: [{ id: email, idProperty: 'email', properties: { email, company: 'Acme Corp', lifecyclestage: 'lead' } }],
})
})
test('with HubSpot as the agent\'s CRM, a captured lead is pushed on its own @crm @workers', async ({ agent, widget, llm, adminApi, request }) => {
await connectCrm(adminApi, 'hubspot', unique('hs-code'))
await adminApi.put(`/agent/${agent.id}/lead-capture`, { data: { enabled: true, require_consent: false, crm_sync_target: 'hubspot' } })
const { email, name } = widget.customer
await llm.script(agent, (s) => s
.when(/@/).captureLead('Thanks! Someone from sales will reach out.', { email, name, company: 'Globex' })
.otherwise().reply('Happy to set that up — what is your email?'))
await widget.embed()
await widget.open()
await widget.send('I would like a demo')
await expect(widget.lastAgentMessage()).toHaveText('Happy to set that up — what is your email?', AI_REPLY)
await widget.send(`Sure, I am ${name}, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Thanks! Someone from sales will reach out.', AI_REPLY)
// Capture queues a sync job; the CRM worker pushes it on its next pass.
const externals = new FakeExternals(request)
expect(await externals.calls('hubspot', 'upsert', { contains: email })).toHaveLength(0)
await runWorkerOnce('crm-sync')
const [upsert] = await externals.calls('hubspot', 'upsert', { contains: email })
expect(upsert.body).toMatchObject({ inputs: [{ id: email, properties: { company: 'Globex', lifecyclestage: 'lead' } }] })
})
})
e2e/specs/crm/pipedrive.org.spec.ts
+57
View File
@@ -0,0 +1,57 @@
import { test, expect, connectCrm, disconnectCrm, findPerson, FakeExternals, syncPersonToCrm, unique, AI_REPLY } from '@e2e'
// One Pipedrive connection per org, so this runs in the serial `org` project and
// disconnects afterwards.
interface PipedriveGrant { access_token: string; api_domain: string }
test.describe('Pipedrive', () => {
test.use({ agentSettings: { leadCapture: true } })
test.afterEach(async ({ adminApi }) => { await disconnectCrm(adminApi, 'pipedrive') })
test('a captured lead becomes a Pipedrive person, organisation and lead with a note @crm', async ({ agent, widget, llm, adminApi, request }) => {
const code = unique('pd-code')
await connectCrm(adminApi, 'pipedrive', code)
const { email, name } = widget.customer
const company = unique('Initech')
await llm.script(agent, (s) => s
.when(/@/).captureLead('Thanks! Someone from sales will reach out.', { email, name, company })
.otherwise().reply('Happy to set that up — what is your email?'))
await widget.embed()
await widget.open()
await widget.send('I would like a demo')
await expect(widget.lastAgentMessage()).toHaveText('Happy to set that up — what is your email?', AI_REPLY)
await widget.send(`Sure, I am ${name}, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Thanks! Someone from sales will reach out.', AI_REPLY)
const person = await findPerson(adminApi, email)
const status = await syncPersonToCrm(adminApi, person.id)
expect(status.synced.map((s) => s.provider)).toContain('pipedrive')
// The install traded the code with the app's own credentials (Basic auth), and
// every API call went to the company's api_domain with the token it was granted.
const externals = new FakeExternals(request)
const [grant] = await externals.calls('pipedrive-oauth', 'token', { contains: code })
const basic = `Basic ${Buffer.from('e2e-pipedrive-client-id:e2e-pipedrive-client-secret').toString('base64')}`
expect(grant.headers.authorization).toBe(basic)
const { access_token: token } = grant.response as PipedriveGrant
const [organization] = await externals.calls('pipedrive', 'organizations', { contains: company })
expect(organization.body).toEqual({ name: company })
const [created] = await externals.calls('pipedrive', 'persons', { contains: email })
expect(created.method).toBe('POST')
expect(created.headers.authorization).toBe(`Bearer ${token}`)
expect(created.body).toMatchObject({
name, emails: [{ value: email, primary: true }], org_id: (organization.response as { data: { id: number } }).data.id,
})
const personId = (created.response as { data: { id: number } }).data.id
const [lead] = (await externals.calls('pipedrive', 'leads', { contains: `"person_id":${personId}` })).filter((c) => c.method === 'POST')
expect(lead.body).toMatchObject({ title: `${name} — ChatterMate lead`, person_id: personId })
const leadId = (lead.response as { data: { id: string } }).data.id
const [note] = await externals.calls('pipedrive', 'notes', { contains: leadId })
expect(note.body).toMatchObject({ lead_id: leadId, content: expect.stringContaining('Lead captured by ChatterMate') })
})
})
e2e/specs/handover/after-hours.org.spec.ts
+26
View File
@@ -0,0 +1,26 @@
import { test, expect, channels, chatsWithAgent, setAlwaysClosed, setAlwaysOpen, unique, AI_REPLY } from '@e2e'
// Changes the org's business hours, so it runs in the serial `org` project and
// reopens the org afterwards (setup leaves it open 24/7).
test.describe('outside business hours', () => {
test.use({ agentSettings: { transferToHuman: true } })
test.afterEach(async ({ adminApi, org }) => { await setAlwaysOpen(adminApi, org.orgId) })
test('a customer asking for a person is told the team will follow up, and the chat waits in the team\'s queue @handover', async ({ agent, llm, adminApi, request, org, handoverTeam }) => {
await setAlwaysClosed(adminApi, org.orgId)
await llm.script(agent, (s) => s.otherwise().handOver('We are closed now; the team will get back to you.'))
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith(unique('Nora'))
await chat.send('I need to talk to someone')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['We are closed now; the team will get back to you.'])
// Queued for the team, unclaimed — whoever opens up next picks it up.
const [conversation] = await chatsWithAgent(adminApi, agent.id)
expect(conversation).toMatchObject({ status: 'transferred', group_id: (await handoverTeam()).groupId, user_id: null })
// The reply was written knowing the team is closed.
const prompts = (await llm.requests(agent)).map((r) => r.lastUser).join('\n')
expect(prompts).toContain('Currently within business hours: False')
})
})
e2e/specs/handover/attachments.spec.ts
+30
View File
@@ -0,0 +1,30 @@
import { test, expect, env, unique, AI_REPLY } from '@e2e'
test.describe('attachments', () => {
test.use({ agentSettings: { transferToHuman: true, allowAttachments: true } })
test('a customer sends a file to the teammate, who can open it — and only through a signed link @handover', async ({ agent, widget, inbox, llm, request }) => {
await llm.script(agent, (s) => s.otherwise().handOver('Let me connect you with a teammate.'))
await widget.embed()
await widget.open()
await widget.send('I need a person')
await widget.shareContactDetails()
await inbox.goto()
await inbox.openConversationWith(widget.customer.name)
await inbox.takeOver()
await inbox.reply('Please send me the receipt.')
await expect(widget.lastAgentMessage()).toHaveText('Please send me the receipt.', AI_REPLY)
const content = `Receipt ${unique('R')} — total 42.00`
await widget.sendFile({ name: 'receipt.txt', mimeType: 'text/plain', content }, 'Here it is')
const link = inbox.attachment('receipt.txt')
await expect(link).toBeVisible(AI_REPLY)
const href = await link.getAttribute('href')
expect((await (await inbox.page.request.get(href!)).text())).toBe(content)
// The stored file is not reachable without the signature.
const unsigned = new URL(href!, 'http://x').pathname
expect((await request.get(`${env.backendUrl}${unsigned}`)).status()).toBe(403)
})
})
e2e/specs/handover/handover.spec.ts
+30
View File
@@ -0,0 +1,30 @@
import { test, expect, expectUnchanged, AI_REPLY } from '@e2e'
test.describe('human handover', () => {
test.use({ agentSettings: { transferToHuman: true } })
test('a customer asks for a person, the teammate\'s reply reaches them, and the AI stays out of it @smoke @handover', async ({ agent, widget, inbox, llm }) => {
await llm.script(agent, (s) => s
.when(/human|person/i).handOver('Let me connect you with a teammate.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('I want to talk to a human')
await widget.shareContactDetails()
await inbox.goto()
await inbox.openConversationWith(widget.customer.name)
await inbox.takeOver()
await inbox.reply('Hi, this is Sam. How can I help?')
await expect(widget.lastAgentMessage()).toHaveText('Hi, this is Sam. How can I help?', AI_REPLY)
const modelCalls = (await llm.requests(agent)).length
await widget.send('My parcel never arrived')
await expect(inbox.lastCustomerMessage()).toContainText('My parcel never arrived')
// A person owns the chat now: the model is not asked, and no bot answer appears.
await expectUnchanged('model calls after takeover', async () => (await llm.requests(agent)).length)
expect((await llm.requests(agent))).toHaveLength(modelCalls)
await expect(widget.lastAgentMessage()).toHaveText('Hi, this is Sam. How can I help?')
})
})
e2e/specs/handover/nobody-online.spec.ts
+20
View File
@@ -0,0 +1,20 @@
import { test, expect, channels, chatsWithAgent, createGroup, createUser, unique, AI_REPLY } from '@e2e'
test('with nobody from the team online, the customer is told the team will follow up, and the chat waits in its queue @handover', async ({ makeAgent, llm, adminApi, request }) => {
// A team whose only member has never signed in, so nobody is online.
const id = unique('offline')
const member = await createUser(adminApi, { name: `Offline ${id}`, email: `${id}@e2e.example.com`, password: 'E2e-Offline-Passw0rd!', role: 'Agent' })
const groupId = await createGroup(adminApi, `Team ${id}`, [member])
const agent = await makeAgent({ transferToHuman: true }, [groupId])
await llm.script(agent, (s) => s.otherwise().handOver('Our team will get back to you shortly.'))
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith(unique('Nora'))
await chat.send('I need to talk to someone')
await expect.poll(() => chat.replies(), AI_REPLY).toEqual(['Our team will get back to you shortly.'])
const [conversation] = await chatsWithAgent(adminApi, agent.id)
expect(conversation).toMatchObject({ status: 'transferred', group_id: groupId, user_id: null })
// The reply was written knowing nobody is online.
const prompts = (await llm.requests(agent)).map((r) => r.lastUser).join('\n')
expect(prompts).toContain('(0 online)')
})
e2e/specs/help-center/publishing.org.spec.ts
+31
View File
@@ -0,0 +1,31 @@
import { test, expect, createArticle, enableHelpCenter, setArticleStatus, unique } from '@e2e'
// Turning the help center on is an org-wide setting, so this runs in the serial
// `org` project. The public site is unauthenticated: exactly what is published,
// nothing else.
test('only published articles are on the public help center @help-center', async ({ adminApi, anonymousPage }) => {
const site = await enableHelpCenter(adminApi)
const code = unique('HC')
const article = await createArticle(adminApi, {
question: `How do I reset my router ${code}?`,
answer: `Hold the reset button for ten seconds. Reference ${code}.`,
})
const articleUrl = `${site}/a/${article.slug}`
// A draft is invisible to the public: not listed, and its URL is not found.
await anonymousPage.goto(site)
await expect(anonymousPage.getByText(code)).toHaveCount(0)
expect((await anonymousPage.request.get(articleUrl)).status()).toBe(404)
await setArticleStatus(adminApi, article.id, 'published')
await anonymousPage.goto(articleUrl)
await expect(anonymousPage.getByRole('heading', { name: `How do I reset my router ${code}?` })).toBeVisible()
await expect(anonymousPage.getByText('Hold the reset button for ten seconds.')).toBeVisible()
await anonymousPage.goto(site)
await expect(anonymousPage.getByText(`How do I reset my router ${code}?`)).toBeVisible()
// Unpublishing takes it down again.
await setArticleStatus(adminApi, article.id, 'draft')
expect((await anonymousPage.request.get(articleUrl)).status()).toBe(404)
})
e2e/specs/integrations/jira.org.spec.ts
+60
View File
@@ -0,0 +1,60 @@
import { test, expect, connectJira, disconnectJira, fakeJira, routeTicketsToJira, FakeExternals, IntegrationsPage, unique, AI_REPLY } from '@e2e'
// One Jira site per org, so this runs in the serial `org` project and disconnects afterwards.
interface JiraGrant { access_token: string }
interface JiraSite { id: string; url: string }
test.describe('Jira', () => {
test.afterEach(async ({ adminApi }) => { await disconnectJira(adminApi) })
test('an admin connects Jira from Settings and the card links to their site @integrations', async ({ page, request }) => {
const since = new Date()
const integrations = new IntegrationsPage(page)
await integrations.goto()
await integrations.connect('Jira')
await expect(page.getByText('Jira connected successfully!')).toBeVisible()
// The code Atlassian's consent screen sent back was traded with the app's own
// credentials, and the site is the one that token reaches.
const externals = new FakeExternals(request)
const [grant] = (await externals.callsSince(since)).filter((c) => c.provider === 'jira-auth' && c.action === 'token')
const [resources] = (await externals.callsSince(since)).filter((c) => c.action === 'accessible-resources')
expect(grant.body).toMatchObject({ grant_type: 'authorization_code', client_id: 'e2e-jira-client-id', redirect_uri: 'http://127.0.0.1:18000/api/v1/jira/callback' })
expect(resources.headers.authorization).toBe(`Bearer ${(grant.response as JiraGrant).access_token}`)
const [site] = resources.response as JiraSite[]
await expect(integrations.card('Jira').getByRole('link', { name: /Visit Jira Site/ })).toHaveAttribute('href', site.url)
})
test('the AI files a Jira issue from a chat, in the project the agent is set up for @integrations', async ({ agent, widget, llm, adminApi, request }) => {
const code = unique('jira-code')
await connectJira(adminApi, code)
await routeTicketsToJira(adminApi, agent.id)
const summary = unique('Refund not received')
await llm.script(agent, (s) => s
.when(/refund/i).callTool('create_jira_ticket', { summary, description: 'Charged twice for one order; wants one payment back.', priority: 'High' })
.thenReply('I have raised this with our billing team.')
.otherwise().reply('How can I help?'))
await widget.embed()
await widget.open()
await widget.send('My refund never arrived')
await expect(widget.lastAgentMessage()).toHaveText('I have raised this with our billing team.', AI_REPLY)
// Jira received the issue in the agent's project and type, at the priority the
// model chose, sent with this org's token.
const externals = new FakeExternals(request)
const [grant] = await externals.calls('jira-auth', 'token', { contains: code })
const [created] = await externals.calls('jira', 'issue', { contains: summary })
expect(created.method).toBe('POST')
expect(created.headers.authorization).toBe(`Bearer ${(grant.response as JiraGrant).access_token}`)
expect(created.body).toMatchObject({
fields: { summary, project: { key: fakeJira.projectKey }, issuetype: { id: fakeJira.taskTypeId }, priority: { id: '2' } },
})
// The model is told the issue key and its link on the connected site.
const { key } = created.response as { key: string }
const toolResults = (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
expect(toolResults).toContain(`.atlassian.net/browse/${key}`)
})
})
e2e/specs/integrations/shopify.org.spec.ts
+62
View File
@@ -0,0 +1,62 @@
import { test, expect, installShopify, uninstallShopify, useShopForAgent, FakeExternals, seedOrg, unique, AI_REPLY } from '@e2e'
// Linking a shop sets the org's domain, so this runs in the serial `org` project and
// uninstalls (restoring the domain) afterwards.
test.describe('Shopify', () => {
let shop: string
let shopId: string | undefined
test.beforeEach(async ({ agent, adminApi }) => {
shop = `${unique('e2e-shop')}.myshopify.com`
shopId = await installShopify(adminApi, shop)
await useShopForAgent(adminApi, agent.id, shopId)
})
test.afterEach(async ({ adminApi, org }) => {
if (shopId) await uninstallShopify(adminApi, shopId, org.orgId, seedOrg.domain)
})
test('a customer asks where their order is and the AI answers from the shop @integrations', async ({ agent, widget, llm, request }) => {
const email = widget.customer.email
await llm.script(agent, (s) => s
.when(/1042/).callTool('get_order_status', { order_number: '#1042', customer_email: email })
.thenReply('Order #1042 has shipped with UPS.')
.otherwise().reply('Happy to check — what is your order number and email?'))
await widget.embed()
await widget.open()
await widget.send('Where is my order?')
await expect(widget.lastAgentMessage()).toHaveText('Happy to check — what is your order number and email?', AI_REPLY)
await widget.send(`It is 1042, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Order #1042 has shipped with UPS.', AI_REPLY)
// The shop was searched for this customer's order with the token its install
// was granted, then the order itself was read.
const externals = new FakeExternals(request)
const [install] = await externals.calls('shopify', 'access_token', { contains: shop })
const [search, read] = await externals.calls('shopify', 'graphql.json', { contains: shop })
expect(search.headers['x-shopify-access-token']).toBe((install.response as { access_token: string }).access_token)
expect(search.body).toMatchObject({ variables: { query: `email:${email.toLowerCase()} AND name:1042` } })
expect(read.body).toMatchObject({ query: expect.stringContaining('query GetOrder') })
// What the model was told: the shipment and its tracking number.
const toolResults = (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
expect(toolResults).toContain('FULFILLED')
expect(toolResults).toMatch(/1ZE2E\d+/)
})
test('an order lookup without the customer\'s email never reaches the shop @integrations', async ({ agent, widget, llm, request }) => {
await llm.script(agent, (s) => s
.otherwise().callTool('get_order_status', { order_number: '1042', customer_email: '' })
.thenReply('Could you tell me the email you ordered with?'))
await widget.embed()
await widget.open()
await widget.send('Where is order 1042?')
await expect(widget.lastAgentMessage()).toHaveText('Could you tell me the email you ordered with?', AI_REPLY)
const externals = new FakeExternals(request)
expect(await externals.calls('shopify', 'graphql.json', { contains: shop })).toHaveLength(0)
const toolResults = (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
expect(toolResults).toContain('requires_user_input')
})
})
e2e/specs/knowledge/knowledge-answer.spec.ts
+27
View File
@@ -0,0 +1,27 @@
import { test, expect, addKnowledgeText, unique, AI_REPLY } from '@e2e'
test.describe('knowledge base', () => {
test('an answer is grounded in text added to the knowledge base @smoke @knowledge', async ({ agent, widget, llm, adminApi, org }) => {
// A token no model could guess, so finding it in the search result proves the
// text was indexed and retrieved — not echoed from the prompt.
const policyCode = unique('RET')
await addKnowledgeText(adminApi, {
orgId: org.orgId,
agentId: agent.id,
title: `Returns policy ${policyCode}`, // titles are unique per org
content: `Returns policy: items can be returned within 45 days of delivery. Quote reference ${policyCode} on the return label.`,
})
await llm.script(agent, (s) => s
.when(/return/i).callTool('search_knowledge_base', { query: 'returns policy' })
.thenReply('You can return items within 45 days of delivery.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('What is your returns policy?')
await expect(widget.lastAgentMessage()).toHaveText('You can return items within 45 days of delivery.', AI_REPLY)
const searchResults = (await llm.requests(agent)).flatMap((request) => request.toolResults)
expect(searchResults.join('\n')).toContain(policyCode)
})
})
e2e/specs/knowledge/knowledge-lifecycle.spec.ts
+41
View File
@@ -0,0 +1,41 @@
import { test, expect, addKnowledgeText, deleteKnowledge, unique, AI_REPLY, type TestAgent, type FakeLlm, type ChatWidget } from '@e2e'
// What the model is shown is what the knowledge base holds for *this* agent, now.
async function askAboutReturns(llm: FakeLlm, agent: TestAgent, widget: ChatWidget): Promise<string> {
await llm.script(agent, (s) => s
.when(/return/i).callTool('search_knowledge_base', { query: 'returns policy' })
.thenReply('Let me check our policy.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('What is your returns policy?')
await expect(widget.lastAgentMessage()).toHaveText('Let me check our policy.', AI_REPLY)
return (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
}
test.describe('knowledge base lifecycle', () => {
test('deleted knowledge is no longer what the AI is shown @knowledge', async ({ agent, widget, llm, adminApi, org, makeWidget }) => {
const code = unique('RET')
const id = await addKnowledgeText(adminApi, {
orgId: org.orgId, agentId: agent.id, title: `Returns policy ${code}`,
content: `Returns policy: items can be returned within 45 days. Reference ${code}.`,
})
expect(await askAboutReturns(llm, agent, widget)).toContain(code)
await deleteKnowledge(adminApi, id)
const freshVisitor = await makeWidget(agent.id)
expect(await askAboutReturns(llm, agent, freshVisitor)).not.toContain(code)
})
test('one agent\'s knowledge never reaches another agent\'s answers @knowledge', async ({ agent, widget, llm, adminApi, org, makeAgent }) => {
const other = await makeAgent()
const secret = unique('INTERNAL')
await addKnowledgeText(adminApi, {
orgId: org.orgId, agentId: other.id, title: `Staff returns rules ${secret}`,
content: `Staff-only returns rules: discount code ${secret} for internal returns.`,
})
// `agent` searches the same org's knowledge base, for the same words.
expect(await askAboutReturns(llm, agent, widget)).not.toContain(secret)
})
})
e2e/specs/knowledge/knowledge-pdf.spec.ts
+24
View File
@@ -0,0 +1,24 @@
import { test, expect, runWorkerOnce, textPdf, unique, unfinishedKnowledge, uploadKnowledgePdf, AI_REPLY, KNOWLEDGE_INDEXED } from '@e2e'
test('an uploaded PDF becomes knowledge the AI answers from @knowledge @workers', async ({ agent, widget, llm, adminApi, org }) => {
const code = unique('WARRANTY')
await uploadKnowledgePdf(adminApi, {
orgId: org.orgId, agentId: agent.id, filename: `${code}.pdf`,
pdf: textPdf(['Warranty policy', `Every product carries a two-year warranty. Reference ${code}.`]),
})
// Production indexes it on the knowledge worker's next poll; here, now.
await runWorkerOnce('knowledge')
// Tests run in parallel: another test's pass may be the one indexing it.
await expect.poll(() => unfinishedKnowledge(adminApi, agent.id), KNOWLEDGE_INDEXED).toEqual([])
await llm.script(agent, (s) => s
.when(/warranty/i).callTool('search_knowledge_base', { query: 'warranty policy' })
.thenReply('Every product has a two-year warranty.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('What warranty do you offer?')
await expect(widget.lastAgentMessage()).toHaveText('Every product has a two-year warranty.', AI_REPLY)
expect((await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')).toContain(code)
})
e2e/specs/leads/lead-capture.spec.ts
+33
View File
@@ -0,0 +1,33 @@
import { test, expect, AI_REPLY } from '@e2e'
interface People { items: Array<{ id: string; email: string | null }> }
interface Person { name: string | null; email: string | null; lead_stage: string; captured_attributes: Record<string, unknown> }
test.describe('lead capture', () => {
test.use({ agentSettings: { leadCapture: true } })
test('a visitor who shares their details shows up as a lead, with what they told us @leads', async ({ agent, widget, llm, peoplePage, adminApi }) => {
const { email, name } = widget.customer
await llm.script(agent, (s) => s
.when(/@/).captureLead('Thanks! Someone from sales will reach out.', { email, name, company: 'Acme Corp' })
.otherwise().reply('Happy to set that up — what is your email?'))
await widget.embed()
await widget.open()
await widget.send('I would like a demo')
// A visitor answers the question once it is asked; replies to back-to-back
// messages are produced concurrently and may arrive in either order.
await expect(widget.lastAgentMessage()).toHaveText('Happy to set that up — what is your email?', AI_REPLY)
await widget.send(`Sure, I am ${name}, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Thanks! Someone from sales will reach out.', AI_REPLY)
await peoplePage.goto()
await peoplePage.findLead(email)
await expect(peoplePage.person(email)).toContainText('Lead')
const [found] = (await adminApi.get<People>('/people', { params: { search: email } })).items
const person = await adminApi.get<Person>(`/people/${found.id}`)
expect(person).toMatchObject({ name, email })
expect(JSON.stringify(person.captured_attributes)).toContain('Acme Corp')
})
})
e2e/specs/leads/people.spec.ts
+58
View File
@@ -0,0 +1,58 @@
import { test, expect, channels, findPerson, phoneNumber, AI_REPLY } from '@e2e'
// The People directory's edit rules, at the API the drawer uses.
test.describe('People', () => {
test('a contact can be renamed, given a phone and marked as a customer @leads', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Thanks for writing in.'))
const chat = (await channels.Email({ api: adminApi, request, agentId: agent.id })).chatWith('Tess')
await chat.send('Hello')
await expect.poll(() => chat.replies(), AI_REPLY).toHaveLength(1)
const person = await findPerson(adminApi, chat.address)
const phone = phoneNumber()
await adminApi.patch(`/people/${person.id}`, { data: { full_name: 'Tess Tester', phone } })
await adminApi.post(`/people/${person.id}/mark-customer`)
expect(await adminApi.get(`/people/${person.id}`)).toMatchObject({ name: 'Tess Tester', phone, lead_stage: 'customer' })
})
test('a phone number must be international and can belong to only one person @leads', async ({ agent, llm, adminApi, request }) => {
await llm.script(agent, (s) => s.otherwise().reply('Thanks for writing in.'))
const inbox = await channels.Email({ api: adminApi, request, agentId: agent.id })
const ann = inbox.chatWith('Ann')
const ben = inbox.chatWith('Ben')
await ann.send('Hello')
await ben.send('Hello')
await expect.poll(() => ann.replies(), AI_REPLY).toHaveLength(1)
await expect.poll(() => ben.replies(), AI_REPLY).toHaveLength(1)
const annPerson = await findPerson(adminApi, ann.address)
const benPerson = await findPerson(adminApi, ben.address)
const local = await adminApi.raw('PATCH', `/people/${annPerson.id}`, { data: { phone: '07700 900123' } })
expect(local.status()).toBe(400)
expect(await local.text()).toContain('international format')
const shared = phoneNumber()
await adminApi.patch(`/people/${annPerson.id}`, { data: { phone: shared } })
const taken = await adminApi.raw('PATCH', `/people/${benPerson.id}`, { data: { phone: shared } })
expect(taken.status()).toBe(400)
expect(await taken.text()).toContain('already belongs to another person')
})
test('once a WhatsApp contact has given a real email, their number can no longer be changed away @leads', async ({ makeAgent, llm, adminApi, request }) => {
// Their phone is now the only thing an incoming WhatsApp message can find them by.
const agent = await makeAgent({ leadCapture: true })
const email = `wes.${Date.now()}@example.com`
await llm.script(agent, (s) => s.otherwise().captureLead('Thanks, noted.', { email, name: 'Wes' }))
const chat = (await channels.WhatsApp({ api: adminApi, request, agentId: agent.id })).chatWith('Wes')
await chat.send(`It is ${email}`)
await expect.poll(() => chat.replies(), AI_REPLY).toHaveLength(1)
const person = await findPerson(adminApi, email)
const response = await adminApi.raw('PATCH', `/people/${person.id}`, { data: { phone: phoneNumber() } })
expect(response.status()).toBe(400)
expect(await response.text()).toContain('how their WhatsApp messages find them')
expect((await adminApi.get<{ phone: string }>(`/people/${person.id}`)).phone).toContain(chat.waId)
})
})
e2e/specs/mcp/connectors.spec.ts
+72
View File
@@ -0,0 +1,72 @@
import { test, expect, attachMcpConnector, createMcpConnector, FakeMcp, unique, AI_REPLY } from '@e2e'
test.describe('MCP connectors', () => {
test('an agent answers with a tool from a connected MCP server @mcp', async ({ agent, widget, llm, adminApi, request }) => {
const mcp = new FakeMcp(request)
const connector = await createMcpConnector(adminApi, { name: unique('orders'), url: mcp.url })
await attachMcpConnector(adminApi, connector.id, agent.id)
const orderId = unique('A')
await llm.script(agent, (s) => s
.when(/order/i).callTool('lookup_order', { order_id: orderId }).thenReply('Your order shipped on 2 May via DHL.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send(`Where is my order ${orderId}?`)
await expect(widget.lastAgentMessage()).toHaveText('Your order shipped on 2 May via DHL.', AI_REPLY)
// The answer came from the MCP server, through the backend, back to the model.
const toolResults = (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
expect(toolResults).toContain(`Order ${orderId}: shipped on 2 May via DHL`)
})
test('a connector\'s secrets are masked when read back, but still reach the server @mcp @security', async ({ agent, widget, llm, adminApi, request }) => {
const mcp = new FakeMcp(request)
const secret = `Bearer ${unique('mcp-secret')}`
const connector = await createMcpConnector(adminApi, { name: unique('orders'), url: mcp.url, headers: { Authorization: secret } })
expect(connector.headers).toEqual({ Authorization: '********' })
expect(JSON.stringify(await adminApi.get(`/mcp-tools/${connector.id}`))).not.toContain(secret)
// Saving the masked value back keeps the real one.
await adminApi.put(`/mcp-tools/${connector.id}`, { data: { headers: { Authorization: '********' } } })
await attachMcpConnector(adminApi, connector.id, agent.id)
await llm.script(agent, (s) => s.otherwise().callTool('lookup_order', { order_id: 'A1' }).thenReply('Checked.'))
await widget.embed()
await widget.open()
await widget.send('Where is my order?')
await expect(widget.lastAgentMessage()).toHaveText('Checked.', AI_REPLY)
expect((await mcp.requests()).map((r) => r.authorization)).toContain(secret)
})
// Nothing listens on port 1: the connection is refused at once.
const DEAD_SERVER = 'http://127.0.0.1:1/mcp'
test('a server that is down costs only its own tools, not the agent\'s other connectors @mcp', async ({ agent, widget, llm, adminApi, request }) => {
const mcp = new FakeMcp(request)
const down = await createMcpConnector(adminApi, { name: unique('down'), url: DEAD_SERVER })
const up = await createMcpConnector(adminApi, { name: unique('orders'), url: mcp.url })
await attachMcpConnector(adminApi, down.id, agent.id)
await attachMcpConnector(adminApi, up.id, agent.id)
const orderId = unique('A')
await llm.script(agent, (s) => s
.when(/order/i).callTool('lookup_order', { order_id: orderId }).thenReply('Your order shipped on 2 May via DHL.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send(`Where is my order ${orderId}?`)
await expect(widget.lastAgentMessage()).toHaveText('Your order shipped on 2 May via DHL.', AI_REPLY)
const toolResults = (await llm.requests(agent)).flatMap((r) => r.toolResults).join('\n')
expect(toolResults).toContain(`Order ${orderId}: shipped on 2 May via DHL`)
})
test('testing a server that is down says it could not connect @mcp', async ({ adminApi }) => {
const down = await createMcpConnector(adminApi, { name: unique('down'), url: DEAD_SERVER })
const result = await adminApi.post<{ success: boolean; error: string | null }>(`/mcp-tools/${down.id}/test`)
expect(result).toMatchObject({ success: false, error: expect.stringContaining('Could not connect to the server') })
})
})
e2e/specs/perf/budgets.spec.ts
+40
View File
@@ -0,0 +1,40 @@
import { test, expect, env, AI_REPLY } from '@e2e'
// Size and speed budgets for what every customer's visitor downloads and waits
// for. About 15% headroom over today; raise a budget on purpose, in review, never
// to make a red run green.
const KB = 1024
const BUDGETS = [
{ file: 'embed loader (chattermate.min.js)', url: `${env.frontendUrl}/webclient/chattermate.min.js`, maxBytes: 60 * KB },
{ file: 'widget script (widget.js)', url: `${env.backendUrl}/assets/widget.js`, maxBytes: 500 * KB },
{ file: 'widget styles (widget.css)', url: `${env.backendUrl}/assets/widget.css`, maxBytes: 135 * KB },
]
test.describe('performance budgets', () => {
for (const { file, url, maxBytes } of BUDGETS) {
test(`${file} stays under ${maxBytes / KB} KB @perf`, async ({ request }) => {
const body = await (await request.get(url, { headers: { 'Accept-Encoding': 'identity' } })).body()
expect(body.length, `${file} is ${Math.round(body.length / KB)} KB`).toBeLessThanOrEqual(maxBytes)
})
}
test('the widget is ready and answers within budget, with an instant model @perf @widget', async ({ agent, llm, widget }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const loadStart = Date.now()
await widget.embed()
await widget.open()
const readyMs = Date.now() - loadStart
// The fake model answers at once, so this is the platform's own overhead.
const sendStart = Date.now()
await widget.send('Hello')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
const replyMs = Date.now() - sendStart
test.info().annotations.push({ type: 'timing', description: `ready ${readyMs} ms, reply ${replyMs} ms` })
expect(readyMs, 'embed → chat ready').toBeLessThan(8_000)
expect(replyMs, 'send → reply').toBeLessThan(5_000)
})
})
e2e/specs/security/agent-socket-origin.spec.ts
+33
View File
@@ -0,0 +1,33 @@
import { test, expect, env, seedOrg, seedUsers, signInCookies, agentSocketOverWebSocket, agentSocketOverPolling } from '@e2e'
// The dashboard's live-chat socket signs in with the sign-in cookies, which
// browsers send on every socket request whatever the page. Sockets get no CORS
// check, so only our own pages may use them: a page on an org's website (an
// unverified signup domain, allowed to call the API) must not get a socket
// signed in as whoever is visiting it.
const DASHBOARD = env.frontendUrl
const ORG_SITE = `https://${seedOrg.domain}`
test.describe('the dashboard live-chat socket', () => {
test('signs in from the dashboard but never from an org\'s website @security', async ({ playwright }) => {
const context = await playwright.request.newContext()
const cookie = await signInCookies(context, seedUsers.admin.email, seedUsers.admin.password)
await context.dispose()
expect(await agentSocketOverWebSocket(DASHBOARD, cookie)).toBe('signed in')
expect(await agentSocketOverWebSocket(ORG_SITE, cookie)).toBe('refused')
})
// Polling needs every request on one worker; the prod-like run has two
test('a <script>-tag (JSONP) handshake carries the cookies but not the page\'s origin, so it never signs in @security @one-worker', async ({ playwright, request }) => {
const context = await playwright.request.newContext()
const cookie = await signInCookies(context, seedUsers.admin.email, seedUsers.admin.password)
await context.dispose()
// The same flow does sign in when the handshake came from the dashboard
expect(await agentSocketOverPolling(request, cookie, ORG_SITE, { Origin: DASHBOARD })).toBe('signed in')
expect(await agentSocketOverPolling(request, cookie, ORG_SITE, {}, '&j=0')).toBe('refused')
expect(await agentSocketOverPolling(request, cookie, ORG_SITE, { 'Sec-Fetch-Site': 'cross-site', 'Sec-Fetch-Dest': 'script' })).toBe('refused')
})
})
e2e/specs/security/headers.spec.ts
+38
View File
@@ -0,0 +1,38 @@
import { test, expect, env, seedUsers, ApiClient } from '@e2e'
// What the backend itself promises about cookies and cross-origin access.
// (Security headers such as HSTS and CSP come from the host's nginx, which is not
// in this repo, so they are checked against a deployment, not here.)
test.describe('security basics', () => {
test('sign-in tokens are HttpOnly, Secure cookies @security', async ({ playwright }) => {
const context = await playwright.request.newContext()
const response = await context.post(`${env.apiUrl}/users/login`, {
form: { username: seedUsers.admin.email, password: seedUsers.admin.password },
})
expect(response.ok()).toBe(true)
const cookies = response.headersArray().filter((h) => h.name.toLowerCase() === 'set-cookie').map((h) => h.value)
for (const name of ['access_token', 'refresh_token']) {
const cookie = cookies.find((c) => c.startsWith(`${name}=`))
expect(cookie, `${name} cookie`).toBeDefined()
expect(cookie).toMatch(/;\s*HttpOnly/i)
expect(cookie).toMatch(/;\s*Secure/i)
}
await context.dispose()
})
test('another website cannot call the API with a visitor\'s credentials @security', async ({ request }) => {
const response = await new ApiClient(request).raw('OPTIONS', '/users/me', {
headers: { Origin: 'https://evil.example', 'Access-Control-Request-Method': 'GET' },
})
expect(response.headers()['access-control-allow-origin']).toBeUndefined()
})
test('the dashboard\'s own origin is allowed @security', async ({ request }) => {
const response = await new ApiClient(request).raw('OPTIONS', '/users/me', {
headers: { Origin: env.frontendUrl, 'Access-Control-Request-Method': 'GET' },
})
expect(response.headers()['access-control-allow-origin']).toBe(env.frontendUrl)
expect(response.headers()['access-control-allow-credentials']).toBe('true')
})
})
e2e/specs/security/org-domain-cors.org.spec.ts
+66
View File
@@ -0,0 +1,66 @@
import { test, expect, env, orgDomain, setOrgDomain, ApiClient } from '@e2e'
// The org's website is what lets its site call the API. Changing it is org-wide,
// so this runs in the serial `org` project and puts the website back. With
// several backend workers the others learn of a change over Redis a moment
// later, so every check polls.
const OLD_SITE = 'https://old-site.e2e.example'
const NEW_SITE = 'https://new-site.e2e.example'
async function allowedOrigin(api: ApiClient, origin: string): Promise<string | undefined> {
// A plain request, not a preflight: every CORS layer adds its header to these,
// so a stale copy of the old origin list would still answer for the old site.
const response = await api.raw('GET', '/organizations/setup-status', { headers: { Origin: origin } })
return response.headers()['access-control-allow-origin']
}
let website: string
test.beforeEach(async ({ adminApi, org }) => { website = await orgDomain(adminApi, org.orgId) })
test.afterEach(async ({ adminApi, org }) => { await setOrgDomain(adminApi, org.orgId, website) })
test.describe('organization website and API access', () => {
test('changing the website moves API access to the new site and revokes the old one @security', async ({ adminApi, org, request }) => {
const anonymous = new ApiClient(request)
await setOrgDomain(adminApi, org.orgId, new URL(OLD_SITE).host)
await expect.poll(() => allowedOrigin(anonymous, OLD_SITE)).toBe(OLD_SITE)
await setOrgDomain(adminApi, org.orgId, new URL(NEW_SITE).host)
await expect.poll(() => allowedOrigin(anonymous, NEW_SITE)).toBe(NEW_SITE)
await expect.poll(() => allowedOrigin(anonymous, OLD_SITE)).toBeUndefined()
// Saving the same website again, then switching back, still allows only the current site
await setOrgDomain(adminApi, org.orgId, new URL(NEW_SITE).host)
await expect.poll(() => allowedOrigin(anonymous, NEW_SITE)).toBe(NEW_SITE)
await setOrgDomain(adminApi, org.orgId, new URL(OLD_SITE).host)
await expect.poll(() => allowedOrigin(anonymous, OLD_SITE)).toBe(OLD_SITE)
await expect.poll(() => allowedOrigin(anonymous, NEW_SITE)).toBeUndefined()
})
})
test.describe('organization website and sign-in cookies', () => {
test('an org\'s website can call the API but never with a signed-in user\'s cookies @security', async ({ adminApi, org, request }) => {
// Any signed-up domain lands in the CORS list, unverified: it must not be
// able to read the dashboard's cookie-authenticated responses
await setOrgDomain(adminApi, org.orgId, new URL(NEW_SITE).host)
const anonymous = new ApiClient(request)
await expect.poll(() => allowedOrigin(anonymous, NEW_SITE)).toBe(NEW_SITE)
const simple = await anonymous.raw('GET', '/users/me', { headers: { Origin: NEW_SITE, Cookie: 'access_token=x' } })
expect(simple.headers()['access-control-allow-origin']).toBe(NEW_SITE)
expect(simple.headers()['access-control-allow-credentials']).toBeUndefined()
const preflight = await anonymous.raw('OPTIONS', '/users/me', {
headers: { Origin: NEW_SITE, 'Access-Control-Request-Method': 'GET' },
})
expect(preflight.headers()['access-control-allow-credentials']).toBeUndefined()
// and the rebuilt CORS layer still lets the dashboard use them
const dashboard = await anonymous.raw('GET', '/users/me', { headers: { Origin: env.frontendUrl } })
expect(dashboard.headers()['access-control-allow-origin']).toBe(env.frontendUrl)
expect(dashboard.headers()['access-control-allow-credentials']).toBe('true')
})
})
e2e/specs/settings/ai-config.org.spec.ts
+47
View File
@@ -0,0 +1,47 @@
import { test, expect, env, seedOrg, currentAiConfig, restoreAi, AI_REPLY, type AiConfig } from '@e2e'
// Changes the org's AI provider, so it runs in the serial `org` project, and puts
// it back afterwards so later specs (and later local runs) see the usual AI.
const fakeModel = { baseUrl: `${env.fakeLlmUrl}/v1`, model: seedOrg.llmModel }
const INVALID_KEY_MESSAGE = 'The provided API key is invalid or does not have access to the selected model.'
test.describe('AI configuration', () => {
let before: AiConfig
test.beforeEach(async ({ adminApi }) => { before = await currentAiConfig(adminApi) })
test.afterEach(async ({ adminApi }) => { await restoreAi(adminApi, before) })
test('a key the provider rejects is reported and the agents keep working @settings', async ({ aiConfigPage, agent, widget, llm }) => {
await aiConfigPage.goto()
await aiConfigPage.useOpenAiCompatible({ ...fakeModel, apiKey: 'e2e-invalid-key' })
await expect(aiConfigPage.error).toHaveText(INVALID_KEY_MESSAGE)
// The rejected key was not saved: a customer still gets answers, through the old key.
await llm.script(agent, (s) => s.otherwise().reply('Still here to help.'))
await widget.embed()
await widget.open()
await widget.send('Hello?')
await expect(widget.lastAgentMessage()).toHaveText('Still here to help.', AI_REPLY)
expect((await llm.requests(agent)).map((r) => r.keySuffix)).not.toContain('id-key')
})
test('a working key is saved and is the one the agents use @settings', async ({ aiConfigPage, agent, widget, llm }) => {
await aiConfigPage.goto()
await aiConfigPage.useOpenAiCompatible({ ...fakeModel, apiKey: 'e2e-fake-key-rotated' })
await expect(aiConfigPage.error).toBeHidden()
await aiConfigPage.goto()
await expect(aiConfigPage.page.getByLabel('Base URL')).toHaveValue(fakeModel.baseUrl)
await llm.script(agent, (s) => s.otherwise().reply('Answered with the new key.'))
await widget.embed()
await widget.open()
await widget.send('Hello?')
await expect(widget.lastAgentMessage()).toHaveText('Answered with the new key.', AI_REPLY)
const lastRequest = (await llm.requests(agent)).at(-1)
expect(lastRequest?.keySuffix).toBe('otated')
expect(lastRequest?.model).toBe(fakeModel.model)
})
})
e2e/specs/setup/org.setup.ts
+47
View File
@@ -0,0 +1,47 @@
import { test as setup, expect } from '@playwright/test'
import { login } from '../../src/api/auth'
import { ApiClient } from '../../src/api/client'
import { isSetUp } from '../../src/api/organizations'
import { seedOrg, seedUsers } from '../../src/env'
import { storageStatePath } from '../../src/org-state'
import { prepareOrg } from '../../src/org-setup'
import { LoginPage } from '../../src/pages/login.page'
import { SetupPage } from '../../src/pages/setup.page'
// OSS edition setup. Runs once per run, before every other project, and writes the
// org contract (src/org-state.ts) that all specs read.
//
// Idempotent: on a fresh database it goes through the real first-run form (which is
// also the test that the form works); on a database left by an earlier run it signs in.
setup('organization, AI and support team are ready', async ({ page, request, browser }) => {
const anonymous = new ApiClient(request)
const { admin } = seedUsers
if (await isSetUp(anonymous)) {
const loginPage = new LoginPage(page)
await loginPage.goto()
await loginPage.signIn(admin.email, admin.password)
await expect(page).not.toHaveURL(/\/login/)
} else {
const setupPage = new SetupPage(page)
await setupPage.goto()
await setupPage.createOrganization({
orgName: seedOrg.name,
domain: seedOrg.domain,
adminName: admin.name,
adminEmail: admin.email,
adminPassword: admin.password,
})
}
const adminStorageState = storageStatePath('admin')
await page.context().storageState({ path: adminStorageState })
const session = await login(anonymous, admin.email, admin.password)
await prepareOrg(browser, {
orgId: session.organizationId,
api: anonymous.withToken(session.token),
adminStorageState,
configureAi: true,
})
})
e2e/specs/team/permissions.spec.ts
+76
View File
@@ -0,0 +1,76 @@
import { test, expect, channels, createGroup, connectEmail, listChannelAccounts, unique, AI_REPLY } from '@e2e'
// What an Agent-role teammate may do, checked at the API — the UI hiding a link
// proves nothing about who the server lets through.
/** Admin-only operations, each with a body the endpoint would accept from an admin. */
const ADMIN_ONLY: Array<{ method: string; path: string; data?: Record<string, unknown> }> = [
{ method: 'GET', path: '/ai/config' },
{ method: 'POST', path: '/ai/setup', data: { model_type: 'OPENAI', model_name: 'gpt-4o', api_key: 'sk-test' } },
{ method: 'GET', path: '/users' },
{ method: 'POST', path: '/users', data: { email: 'blocked@e2e.example.com', full_name: 'Blocked', password: 'E2e-Blocked-Passw0rd!', role_id: 1 } },
{ method: 'GET', path: '/roles' },
{ method: 'GET', path: '/groups' },
{ method: 'POST', path: '/groups', data: { name: 'Blocked group' } },
{ method: 'GET', path: '/agent/list' },
{ method: 'POST', path: '/agent', data: { name: 'Blocked agent', instructions: ['x'] } },
{ method: 'GET', path: '/widget-apps' },
{ method: 'GET', path: '/tickets/settings' },
{ method: 'PUT', path: '/tickets/settings', data: {} },
{ method: 'POST', path: '/channels/telegram', data: { bot_token: '1:blocked' } },
{ method: 'GET', path: '/mcp-tools' },
]
test.describe('Agent role', () => {
test('is refused every admin operation by the API @team', async ({ makeMember }) => {
const agent = await makeMember({ role: 'Agent' })
const refused = await Promise.all(ADMIN_ONLY.map(async ({ method, path, data }) =>
`${method} ${path} → ${(await agent.api.raw(method, path, { data })).status()}`))
expect(refused).toEqual(ADMIN_ONLY.map(({ method, path }) => `${method} ${path} → 403`))
// The same token works where an agent belongs.
expect((await agent.api.raw('GET', '/chats/recent')).status()).toBe(200)
})
test('sees the connected channels for the inbox, but never their webhook secrets @team', async ({ adminApi, makeMember }) => {
const inbox = await connectEmail(adminApi, `${unique('support')}@e2e.example.com`)
expect(inbox.webhook_url).toContain('token=')
const agent = await makeMember({ role: 'Agent' })
const seen = (await listChannelAccounts(agent.api)).find((a) => a.id === inbox.id)
expect(seen).toBeDefined()
expect(seen?.webhook_url ?? null).toBeNull()
})
test('with "assigned only" sees their own group\'s handed-over chats and not another group\'s @team @handover', async ({ adminApi, makeMember, makeAgent, llm, request }) => {
// Two teams, each with an AI agent that hands over to it.
const groupA = await createGroup(adminApi, unique('Team A'), [])
const groupB = await createGroup(adminApi, unique('Team B'), [])
const alice = await makeMember({ role: 'Agent', groupIds: [groupA], seeAllAiChats: false })
await makeMember({ role: 'Agent', groupIds: [groupB] }) // someone online in B, so B's handover happens
const agentA = await makeAgent({ transferToHuman: true }, [groupA])
const agentB = await makeAgent({ transferToHuman: true }, [groupB])
for (const agent of [agentA, agentB]) {
await llm.script(agent, (s) => s.otherwise().handOver('Let me connect you with a teammate.'))
}
const forA = (await channels.WhatsApp({ api: adminApi, request, agentId: agentA.id })).chatWith(unique('Ana'))
const forB = (await channels.WhatsApp({ api: adminApi, request, agentId: agentB.id })).chatWith(unique('Ben'))
await forA.send('I need a person')
await forB.send('I need a person')
await expect.poll(() => forA.replies(), AI_REPLY).toHaveLength(1)
await expect.poll(() => forB.replies(), AI_REPLY).toHaveLength(1)
const visibleTo = async () => (await alice.api.get<Array<{ agent: { id: string }; session_id: string }>>('/chats/recent', { params: { limit: 100 } }))
const agentsAliceSees = (await visibleTo()).map((c) => String(c.agent.id))
expect(agentsAliceSees).toContain(agentA.id)
expect(agentsAliceSees).not.toContain(agentB.id)
// Opening B's chat directly is refused too, not just hidden from the list.
const allChats = await adminApi.get<Array<{ agent: { id: string }; session_id: string }>>('/chats/recent', { params: { limit: 100, agent_id: agentB.id } })
const bSession = allChats.find((c) => String(c.agent.id) === agentB.id)!.session_id
expect([403, 404]).toContain((await alice.api.raw('GET', `/chats/${bSession}`)).status())
})
})
e2e/specs/team/team.spec.ts
+31
View File
@@ -0,0 +1,31 @@
import { test, expect, storageStatePath, unique, LoginPage, Sidebar } from '@e2e'
test.describe('team', () => {
test('an admin adds an agent who can sign in to the inbox @team', async ({ humanAgentsPage, anonymousPage }) => {
const id = unique('agent')
const member = { name: `Alex ${id}`, email: `${id}@e2e.example.com`, password: 'E2e-New-Agent-Passw0rd!', role: 'Agent' as const }
await humanAgentsPage.goto()
await humanAgentsPage.invite(member)
await expect(humanAgentsPage.member(member.email)).toBeVisible()
const login = new LoginPage(anonymousPage)
await login.goto()
await login.signIn(member.email, member.password)
await expect(anonymousPage).toHaveURL(/\/conversations/)
})
})
test.describe('as a human agent', () => {
test.use({ storageState: storageStatePath('human-agent') })
test('the Agent role does not see admin areas in the navigation @team', async ({ page }) => {
// Not /conversations: the inbox collapses the sidebar to icons.
await page.goto('/people')
const sidebar = new Sidebar(page)
await expect(sidebar.link('Inbox')).toBeVisible()
for (const adminArea of ['AI Agents', 'Human Agents', 'Knowledge', 'AI Configuration', 'Organization']) {
await expect(sidebar.link(adminArea)).toBeHidden()
}
})
})
e2e/specs/tickets/ai-ticket.spec.ts
+83
View File
@@ -0,0 +1,83 @@
import { test, expect, chatMessages, chatsWithAgent, runWorkerOnce, sql, unique, uuid, AI_REPLY } from '@e2e'
test.describe('tickets', () => {
test('the AI opens a ticket that an agent then works in the dashboard @tickets', async ({ agent, widget, llm, ticketsPage }) => {
const summary = `Checkout button does nothing ${unique('T')}`
await llm.script(agent, (s) => s
.when(/broken|does nothing/i).createTicket('I have logged this for our team.', summary)
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('The checkout button does nothing when I click it')
await expect(widget.lastAgentMessage()).toHaveText('I have logged this for our team.', AI_REPLY)
await ticketsPage.goto()
await ticketsPage.open(summary)
await ticketsPage.setStatus('In progress')
await ticketsPage.page.reload()
await expect(ticketsPage.statusLabel).toHaveText('In progress')
})
})
test.describe('resolving a ticket', () => {
test('the customer hears about the fix in the chat the ticket came from @tickets', async ({ agent, widget, llm, ticketsPage, adminApi }) => {
const summary = `Coupon code rejected ${unique('T')}`
await llm.script(agent, (s) => s
.when(/coupon/i).createTicket('I have logged this for our team.', summary)
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('My coupon code is rejected at checkout')
await expect(widget.lastAgentMessage()).toHaveText('I have logged this for our team.', AI_REPLY)
await ticketsPage.goto()
await ticketsPage.open(summary)
const ticketId = new URL(ticketsPage.page.url()).pathname.split('/').at(-1)
await adminApi.post(`/tickets/${ticketId}/resolve`, {
data: { outcome: 'fixed', resolution_summary: 'Expired promotion table', customer_message: 'Good news: your coupon works again.' },
})
await expect(widget.lastAgentMessage()).toContainText('Good news: your coupon works again.', AI_REPLY)
await ticketsPage.page.reload()
await expect(ticketsPage.statusLabel).toHaveText('Resolved · confirming')
})
})
// The rating request comes from the ticket worker, a separate process; it reaches
// the visitor's open widget through Redis, as in production — so @redis (nightly).
test.describe('ticket satisfaction', () => {
test.use({ agentSettings: { askForRating: true } })
test('a resolved ticket closes after its confirmation window and asks the customer for a rating, which is kept @tickets @workers @redis', async ({ agent, widget, llm, ticketsPage, adminApi }) => {
const summary = `Invoice shows the wrong VAT ${unique('T')}`
await llm.script(agent, (s) => s
.when(/invoice/i).createTicket('I have logged this for our team.', summary)
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('My invoice shows the wrong VAT')
await expect(widget.lastAgentMessage()).toHaveText('I have logged this for our team.', AI_REPLY)
await ticketsPage.goto()
await ticketsPage.open(summary)
const ticketId = uuid(new URL(ticketsPage.page.url()).pathname.split('/').at(-1)!)
await adminApi.post(`/tickets/${ticketId}/resolve`, { data: { outcome: 'fixed', customer_message: 'Fixed: a corrected invoice is on its way.' } })
await expect(widget.lastAgentMessage()).toContainText('Fixed: a corrected invoice is on its way.', AI_REPLY)
// The customer never replies; the confirmation window (72h by default) runs out.
sql(`UPDATE tickets SET confirmation_requested_at = now() - interval '73 hours' WHERE id = '${ticketId}'`)
await runWorkerOnce('ticket-lifecycle')
await expect(widget.ratingPrompt).toBeVisible(AI_REPLY)
await widget.rate(4)
await expect(widget.ratingThanks).toBeVisible()
await expect.poll(async () => (await adminApi.get<{ ticket: { status: string; csat_score: number | null } }>(`/tickets/${ticketId}`)).ticket)
.toMatchObject({ status: 'closed', csat_score: 4 })
// The ask is part of the conversation, so agents see it in the thread too.
const [chat] = await chatsWithAgent(adminApi, agent.id)
const asks = (await chatMessages(adminApi, chat.session_id)).filter((m) => m.message?.startsWith('How did we do on ticket'))
expect(asks).toHaveLength(1)
})
})
e2e/specs/visual/widget.spec.ts
+25
View File
@@ -0,0 +1,25 @@
import { test, expect, customizeAgent } from '@e2e'
// Screenshot baselines are generated on Linux in CI (the nightly job, or the e2e
// workflow run by hand with "update snapshots") and committed from its artifact.
// Never commit a baseline made on a Mac: fonts render differently.
test.describe('widget appearance', () => {
for (const scheme of ['light', 'dark'] as const) {
test(`launcher and open chat, ${scheme} @visual @widget`, async ({ agent, adminApi, widget }) => {
// Fixed name and text, so nothing in the shot changes from run to run (the
// avatar is drawn from the name, too).
await adminApi.put(`/agent/${agent.id}`, { data: { display_name: 'Acme Support' } })
await customizeAgent(adminApi, agent.id, { welcome_message: 'Hi! How can we help today?' })
await widget.page.emulateMedia({ colorScheme: scheme })
await widget.embed()
await expect(widget.launcher).toBeVisible()
await expect(widget.launcher).toHaveScreenshot(`launcher-${scheme}.png`)
await widget.open()
await expect(widget.welcomeMessage).toBeVisible()
await expect(widget.agentName).toHaveText('Acme Support')
await expect(widget.panel).toHaveScreenshot(`chat-${scheme}.png`)
})
}
})
e2e/specs/widget/ai-failures.spec.ts
+49
View File
@@ -0,0 +1,49 @@
import { test, expect, AI_REPLY } from '@e2e'
// When the model fails, the customer gets an apology rather than silence or debris,
// and the very next message is answered normally.
const APOLOGY = 'I apologize, but I encountered an error, please try again later.'
test.describe('when the AI provider fails', () => {
test('an error from the provider becomes an apology, and the chat carries on @widget', async ({ agent, llm, widget }) => {
await llm.script(agent, (s) => s
.when(/broken/).fail(500, 'upstream exploded')
.otherwise().reply('Back to normal.'))
await widget.embed()
await widget.open()
await widget.send('this is broken')
await expect(widget.lastAgentMessage()).toHaveText(APOLOGY, AI_REPLY)
await widget.send('hello again')
await expect(widget.lastAgentMessage()).toHaveText('Back to normal.', AI_REPLY)
})
test('a provider that never answers is given up on, with an apology @widget @slow', async ({ agent, llm, widget }) => {
// Longer than AGENT_RUN_TIMEOUT (.env.e2e), so the backend stops waiting.
await llm.script(agent, (s) => s
.when(/broken/).after(25_000).reply('Far too late.')
.otherwise().reply('Back to normal.'))
await widget.embed()
await widget.open()
await widget.send('this is broken')
await expect(widget.lastAgentMessage()).toHaveText(APOLOGY, AI_REPLY)
await widget.send('hello again')
await expect(widget.lastAgentMessage()).toHaveText('Back to normal.', AI_REPLY)
await expect(widget.agentMessages.filter({ hasText: 'Far too late.' })).toHaveCount(0)
})
test('a reply that is not the JSON asked for never reaches the customer raw @widget', async ({ agent, llm, widget }) => {
await llm.script(agent, (s) => s
.when(/broken/).malformed()
.otherwise().reply('Back to normal.'))
await widget.embed()
await widget.open()
await widget.send('this is broken')
await expect(widget.lastAgentMessage()).toHaveText(APOLOGY, AI_REPLY)
await widget.send('hello again')
await expect(widget.lastAgentMessage()).toHaveText('Back to normal.', AI_REPLY)
})
})
e2e/specs/widget/chat-styles.spec.ts
+58
View File
@@ -0,0 +1,58 @@
import { test, expect, AI_REPLY, customizeAgent, findPerson } from '@e2e'
// Each way a visitor reaches the chat, and each again when the first answer that
// binds them to a customer goes missing (the socket then gets the page's anonymous
// token and the widget has to authorize again).
for (const lost of [false, true]) {
const when = lost ? ', even when its first authorization goes missing' : ''
test.describe(`chat styles${lost ? ' (first authorization lost)' : ''}`, () => {
test.beforeEach(async ({ widget }) => {
if (lost) await widget.loseFirstAuthorization()
})
test(`an email-first agent: the visitor gives an email and gets the answer${when} @widget`, async ({ agent, widget, llm, adminApi }) => {
await customizeAgent(adminApi, agent.id, { collect_email: true })
await llm.script(agent, (s) => s.otherwise().reply('Thanks, how can I help?'))
await widget.embed()
await widget.openToEmailGate()
await widget.giveEmail()
await widget.send('Hello')
await expect(widget.lastAgentMessage()).toHaveText('Thanks, how can I help?', AI_REPLY)
expect((await findPerson(adminApi, widget.customer.email)).email).toBe(widget.customer.email)
})
test(`an Ask anything agent answers in its panel, with no email asked${when} @widget`, async ({ agent, widget, llm, adminApi }) => {
await customizeAgent(adminApi, agent.id, { chat_style: 'ASK_ANYTHING' })
await llm.script(agent, (s) => s.otherwise().reply('We ship worldwide.'))
await widget.embed()
await widget.openAskAnything()
await widget.ask('Do you ship abroad?')
await expect(widget.lastAnswer()).toHaveText('We ship worldwide.', AI_REPLY)
})
test.describe('lead capture', () => {
test.use({ agentSettings: { leadCapture: true } })
test(`a visitor with no email asked shares details and becomes a lead${when} @widget @leads`, async ({ agent, widget, llm, adminApi }) => {
const { email, name } = widget.customer
await llm.script(agent, (s) => s
.when(/@/).captureLead('Thanks! Someone from sales will reach out.', { email, name })
.otherwise().reply('Happy to help — what is your email?'))
await widget.embed()
await widget.open()
await widget.send('I want a demo')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help — what is your email?', AI_REPLY)
await widget.send(`I am ${name}, ${email}`)
await expect(widget.lastAgentMessage()).toHaveText('Thanks! Someone from sales will reach out.', AI_REPLY)
expect((await findPerson(adminApi, email)).lead_stage).toBe('lead')
})
})
})
}
e2e/specs/widget/conversation-ownership.spec.ts
+148
View File
@@ -0,0 +1,148 @@
import { test, expect, AI_REPLY, createWidgetApp, customizeAgent, findPerson, issueConversationToken, unique } from '@e2e'
// An email typed into the widget is not proof of being that customer. Someone else
// who types it gets the same person in the inbox, but never their conversations,
// and never the right to change who they are.
test.describe('conversation ownership', () => {
test.beforeEach(async ({ agent, adminApi }) => {
await customizeAgent(adminApi, agent.id, { collect_email: true })
})
test('a second browser that gives a customer\'s email starts its own conversation @widget @security', async ({ agent, widget, makeVisitor, llm }) => {
await llm.script(agent, (s) => s
.when(/order/i).reply('Order 1234 ships tomorrow.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.openToEmailGate()
await widget.giveEmail()
await widget.send('Where is my order 1234?')
await expect(widget.lastAgentMessage()).toHaveText('Order 1234 ships tomorrow.', AI_REPLY)
const stranger = await makeVisitor(widget.customer)
await stranger.embed()
await stranger.openToEmailGate()
await stranger.giveEmail()
await stranger.send('Hello')
await expect(stranger.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
// Their reply has arrived, so the history has loaded: only their own message is in it.
await expect(stranger.customerMessages).toHaveText(['Hello'])
await widget.page.reload()
await widget.open()
await expect(widget.lastAgentMessage()).toHaveText('Order 1234 ships tomorrow.', AI_REPLY)
await expect(widget.customerMessages).toHaveText(['Where is my order 1234?'])
})
test.describe('lead capture', () => {
test.use({ agentSettings: { leadCapture: true } })
test('a visitor who only typed a customer\'s email cannot rename them or make them a lead @widget @security @leads', async ({ agent, widget, makeVisitor, llm, adminApi }) => {
const { email } = widget.customer
await llm.script(agent, (s) => s
.when(/Someone Else/).captureLead('Thanks, noted.', { email, name: 'Someone Else' })
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.openToEmailGate()
await widget.giveEmail()
await widget.send('Hello')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
const stranger = await makeVisitor(widget.customer)
await stranger.embed()
await stranger.openToEmailGate()
await stranger.giveEmail()
await stranger.send(`I am Someone Else, ${email}`)
await expect(stranger.lastAgentMessage()).toHaveText('Thanks, noted.', AI_REPLY)
const person = await findPerson(adminApi, email)
expect(person.name).not.toBe('Someone Else')
expect(person.lead_stage).toBe('visitor')
})
})
test('an address claimed before its owner signs in gives the claimer none of their chats @widget @security', async ({ agent, widget, makeVisitor, llm, adminApi }) => {
await llm.script(agent, (s) => s
.when(/invoice/i).reply('Your invoice is on its way.')
.otherwise().reply('Happy to help!'))
const owner = { name: `Owner ${unique('owner')}`, email: `${unique('owner')}@example.com` }
// Someone claims the address first, through the widget's email gate.
const claimer = await makeVisitor(owner)
await claimer.embed()
await claimer.openToEmailGate()
await claimer.giveEmail()
await claimer.send('Hello')
await expect(claimer.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
// Then the site signs its real owner in, and they chat.
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
const token = await issueConversationToken(adminApi, apiKey, widget.widgetId, owner)
await widget.embed({ token })
await widget.open()
await widget.send('Where is my invoice?')
await expect(widget.lastAgentMessage()).toHaveText('Your invoice is on its way.', AI_REPLY)
// The claimer comes back: still able to chat, but in a conversation of their own.
await claimer.page.reload()
await claimer.open()
await claimer.send('Anything new?')
await expect(claimer.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
await expect(claimer.customerMessages).toHaveText(['Anything new?'])
await widget.page.reload()
await widget.open()
await expect(widget.customerMessages).toHaveText(['Where is my invoice?'])
})
})
// What scoping a conversation to its browser must not take away.
test.describe('a visitor keeps their own conversation', () => {
test('a second tab of the same browser joins the same conversation @widget', async ({ agent, widget, llm }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('Hello from the first tab')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
const tab = await widget.inAnotherTab()
await tab.embed()
await tab.open()
await expect(tab.customerMessages).toHaveText(['Hello from the first tab'])
})
test('a visitor the site signed in finds their conversation in another browser @widget', async ({ agent, widget, makeVisitor, llm, adminApi }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
await widget.embed({ token: await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer) })
await widget.open()
await widget.send('Started on my laptop')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
const phone = await makeVisitor(widget.customer)
await phone.embed({ token: await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer) })
await phone.open()
await expect(phone.customerMessages).toHaveText(['Started on my laptop'])
})
test('a site that signs the visitor in with identify() hands the widget their conversation @widget', async ({ agent, widget, makeVisitor, llm, adminApi }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
await widget.embed({ token: await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer) })
await widget.open()
await widget.send('Signed-in question')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
// The same person opens the site signed out, then signs in on the page.
const later = await makeVisitor(widget.customer)
await later.embed()
await later.open()
await later.api('identify', await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer))
await later.open()
await expect(later.customerMessages).toHaveText(['Signed-in question'])
})
})
e2e/specs/widget/iframe-snippet.spec.ts
+14
View File
@@ -0,0 +1,14 @@
import { test, expect, AI_REPLY } from '@e2e'
// The dashboard also offers an iframe snippet: the chat page itself, served by
// the API. Its socket comes from the API's own origin, which must be allowed
// whether or not the operator listed it in CORS_ORIGINS.
test('a customer chats through the iframe snippet @widget', async ({ agent, widget, llm }) => {
await llm.script(agent, (s) => s.otherwise().reply('Hello from the iframe'))
await widget.embed({ snippet: 'iframe' })
await widget.expectReady()
await widget.send('Hi')
await expect(widget.lastAgentMessage()).toHaveText('Hello from the iframe', AI_REPLY)
})
e2e/specs/widget/js-api.spec.ts
+25
View File
@@ -0,0 +1,25 @@
import { test, expect } from '@e2e'
test.describe('widget JS API', () => {
test('the host page opens and closes the chat with window.ChatterMate @widget', async ({ widget }) => {
await widget.embed()
await widget.api('open')
await expect(widget.panel).toHaveClass(/\bactive\b/)
await widget.expectReady()
expect(await widget.api('isOpen')).toBe(true)
await widget.api('close')
await expect(widget.panel).not.toHaveClass(/\bactive\b/)
expect(await widget.api('isOpen')).toBe(false)
})
test('a site can hide the launcher and open the chat from its own button @widget', async ({ widget }) => {
await widget.embed({ config: { launcher: false }, withTriggerButton: true })
await expect(widget.launcher).toBeHidden()
await widget.hostChatButton.click()
await expect(widget.panel).toHaveClass(/\bactive\b/)
await widget.expectReady()
})
})
e2e/specs/widget/placement.spec.ts
+49
View File
@@ -0,0 +1,49 @@
import { test, expect, customizeAgent, expectUnchanged } from '@e2e'
// Where the widget sits and how it looks: the site's own config, the dashboard's
// display settings, and which one wins.
test.describe('widget placement', () => {
test('the launcher goes on the left when the site asks for it @widget', async ({ widget }) => {
await widget.embed({ config: { position: { side: 'left' } } })
const box = (await widget.launcher.boundingBox())!
expect(box.x).toBeLessThan(100)
})
// The suite runs with reduced motion, which the loader honours by not sliding the
// panel in, so its final position is there at once.
test('sidebar mode opens the chat as a full-height panel on the right @widget', async ({ widget }) => {
await widget.embed({ config: { displayMode: 'sidebar-right' } })
await widget.open()
const viewport = widget.page.viewportSize()!
const panel = (await widget.panel.boundingBox())!
expect(panel.height).toBeGreaterThanOrEqual(viewport.height - 1)
expect(Math.round(panel.x + panel.width)).toBe(viewport.width)
})
test('the dashboard\'s display settings apply without any code, and the site\'s own config wins @widget', async ({ agent, adminApi, makeWidget }) => {
await customizeAgent(adminApi, agent.id, { customization_metadata: { widget_display: { side: 'left' } } })
// The loader applies the dashboard's settings once it has fetched them.
const fromDashboard = await makeWidget(agent.id)
await fromDashboard.embed()
await expect.poll(async () => (await fromDashboard.launcher.boundingBox())?.x).toBeLessThan(100)
const siteOverrides = await makeWidget(agent.id)
await siteOverrides.embed({ config: { position: { side: 'right' } } })
const viewport = siteOverrides.page.viewportSize()!
await expect(siteOverrides.launcher).toBeVisible()
await expect.poll(async () => (await siteOverrides.launcher.boundingBox())?.x).toBeGreaterThan(viewport.width - 150)
// …and stays there once the dashboard's settings arrive.
await expectUnchanged('launcher position', async () => Math.round((await siteOverrides.launcher.boundingBox())!.x))
})
test('the agent\'s welcome message and launcher colour show in the widget @widget', async ({ agent, adminApi, widget }) => {
await customizeAgent(adminApi, agent.id, { welcome_message: 'Hi from Acme support — how can we help?', chat_bubble_color: '#123456' })
await widget.embed()
await expect(widget.launcher).toHaveCSS('background-color', 'rgb(18, 52, 86)')
await widget.open()
await expect(widget.welcomeMessage).toHaveText('Hi from Acme support — how can we help?')
})
})
e2e/specs/widget/rating.spec.ts
+26
View File
@@ -0,0 +1,26 @@
import { test, expect } from '@e2e'
interface AgentPerformance { bot_agents: Array<{ id: string; avg_rating: number; rating_count: number }> }
test.describe('chat rating', () => {
test.use({ agentSettings: { askForRating: true } })
test('a customer rates the chat when the agent wraps up, and the rating is recorded against the agent @widget', async ({ agent, widget, llm, adminApi }) => {
await llm.script(agent, (s) => s
.when(/that is all|thanks/i).endChat('Glad I could help. Goodbye!')
.otherwise().reply('Sure thing.'))
await widget.embed()
await widget.open()
await widget.send('That is all, thanks')
await expect(widget.ratingPrompt).toBeVisible()
await widget.rate(4)
await expect(widget.ratingThanks).toBeVisible()
// The thanks is client-side; the agent's numbers are what the team actually sees.
const ratingOf = async () => (await adminApi.get<AgentPerformance>('/analytics/agent-performance', { params: { time_range: '24h' } }))
.bot_agents.find((a) => a.id === agent.id)
await expect.poll(ratingOf).toMatchObject({ rating_count: 1, avg_rating: 4 })
})
})
e2e/specs/widget/token-auth.spec.ts
+61
View File
@@ -0,0 +1,61 @@
import { test, expect, ApiClient, createWidgetApp, issueConversationToken, unique, AI_REPLY } from '@e2e'
test.describe('token-authenticated widget', () => {
test.use({ agentSettings: { requireTokenAuth: true } })
test('a visitor with a token from the site\'s server can chat @widget', async ({ agent, widget, llm, adminApi }) => {
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
const token = await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer)
await llm.script(agent, (s) => s.otherwise().reply('Welcome back!'))
await widget.embed({ token })
await widget.open()
await widget.send('Hi')
await expect(widget.lastAgentMessage()).toHaveText('Welcome back!', AI_REPLY)
})
test('without a token the widget refuses to load @widget', async ({ widget }) => {
await widget.embed()
// The notice lives in the chat panel, so the visitor sees it when they click the launcher.
await widget.launcher.click()
await expect(widget.unavailableNotice).toBeVisible()
})
test('a tampered token is refused @widget @security', async ({ widget, adminApi }) => {
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
const token = await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer)
// Swap the signature for another token's: a well-formed JWT this widget never signed.
const other = await issueConversationToken(adminApi, apiKey, widget.widgetId, { ...widget.customer, email: `x.${widget.customer.email}` })
const tampered = `${token.split('.').slice(0, 2).join('.')}.${other.split('.')[2]}`
await widget.embed({ token: tampered })
await widget.launcher.click()
await expect(widget.unavailableNotice).toBeVisible()
})
test('a token minted for another widget is refused @widget @security', async ({ widget, adminApi, agent, makeWidget }) => {
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
const otherWidget = await makeWidget(agent.id)
const tokenForOther = await issueConversationToken(adminApi, apiKey, otherWidget.widgetId, widget.customer)
await widget.embed({ token: tokenForOther })
await widget.launcher.click()
await expect(widget.unavailableNotice).toBeVisible()
})
test('an expired token no longer opens the chat @widget @security @slow', async ({ widget, adminApi, request }) => {
test.slow()
const { apiKey } = await createWidgetApp(adminApi, unique('acme-site'))
const token = await issueConversationToken(adminApi, apiKey, widget.widgetId, widget.customer, 60)
// What the loader fetches to render the chat frame for this visitor. Each
// poll on a fresh connection: the backend drops a keep-alive connection
// idle for 5s, the poll interval, and a reused one can hang up mid-request.
const visitor = new ApiClient(request, token)
const frame = () => visitor.raw('GET', `/widgets/${widget.widgetId}/data`, { headers: { Connection: 'close' } })
expect((await frame()).status()).toBe(200)
// The shortest lifetime the API allows; the backend's own clock decides.
await expect.poll(async () => (await frame()).status(), { timeout: 90_000, intervals: [5_000] }).toBe(401)
expect(JSON.stringify(await (await frame()).json())).toContain('Conversation token expired')
})
})
e2e/specs/widget/widget-chat.spec.ts
+41
View File
@@ -0,0 +1,41 @@
import { test, expect, AI_REPLY } from '@e2e'
test.describe('widget chat', () => {
test('a customer gets the agent\'s answer @smoke @widget', async ({ agent, widget, llm }) => {
await llm.script(agent, (s) => s
.when(/opening hours/i).reply('We are open 9am to 5pm, Monday to Friday.')
.otherwise().reply('Happy to help!'))
await widget.embed()
await widget.open()
await widget.send('What are your opening hours?')
await expect(widget.lastAgentMessage()).toHaveText('We are open 9am to 5pm, Monday to Friday.', AI_REPLY)
})
test('the conversation is still there after the customer reloads the page @widget', async ({ agent, widget, llm }) => {
await llm.script(agent, (s) => s.otherwise().reply('Your order ships tomorrow.'))
await widget.embed()
await widget.open()
await widget.send('Where is my order?')
await expect(widget.lastAgentMessage()).toHaveText('Your order ships tomorrow.', AI_REPLY)
await widget.page.reload()
await widget.open()
await expect(widget.customerMessages.last()).toHaveText('Where is my order?')
await expect(widget.lastAgentMessage()).toHaveText('Your order ships tomorrow.', AI_REPLY)
})
test('a visitor whose first authorization went missing can still chat @widget', async ({ agent, widget, llm }) => {
await llm.script(agent, (s) => s.otherwise().reply('Happy to help!'))
// The socket refuses the anonymous token; the widget authorizes again and connects.
await widget.loseFirstAuthorization()
await widget.embed()
await widget.open()
await widget.send('Hello?')
await expect(widget.lastAgentMessage()).toHaveText('Happy to help!', AI_REPLY)
})
})
e2e/specs/workflows/workflows.spec.ts
+107
View File
@@ -0,0 +1,107 @@
import { test, expect, expectUnchanged, publishWorkflow, AI_REPLY, type WorkflowGraph } from '@e2e'
// A published workflow runs instead of the model: scripted steps, no AI calls.
test.describe('workflows', () => {
test('a one-message workflow greets once on load and answers each message once, without the AI @workflows', async ({ agent, adminApi, widget, llm }) => {
await publishWorkflow(adminApi, agent.id, {
nodes: [{ id: 'msg-1', node_type: 'message', name: 'Greeting', config: { message_text: 'Hello from our workflow.' } }],
})
await widget.embed()
await widget.open()
await expect(widget.agentMessages).toHaveText(['Hello from our workflow.'], AI_REPLY)
await widget.send('hi')
await expect(widget.agentMessages).toHaveText(['Hello from our workflow.', 'Hello from our workflow.'], AI_REPLY)
await expectUnchanged('agent messages', () => widget.agentMessages.allTextContents())
expect(await llm.requests(agent)).toHaveLength(0)
})
test('a workflow asks a question and branches on the answer @workflows', async ({ agent, adminApi, widget }) => {
await publishWorkflow(adminApi, agent.id, {
nodes: [
{ id: 'ask', node_type: 'user_input', name: 'Ask Plan', config: { prompt_message: 'Which plan are you on? (pro/basic)', confirmation_message: 'Thanks — {{ask_plan_input}} it is.' } },
{ id: 'is-pro', node_type: 'condition', name: 'Is Pro', config: { condition_expression: '{{ask_plan_input}} === pro' } },
{ id: 'pro', node_type: 'message', name: 'Pro', config: { message_text: 'Pro customers get priority support.' } },
{ id: 'basic', node_type: 'message', name: 'Basic', config: { message_text: 'Basic customers can upgrade any time.' } },
],
connections: [
{ id: 'c1', source_node_id: 'ask', target_node_id: 'is-pro' },
{ id: 'c2', source_node_id: 'is-pro', target_node_id: 'pro', label: 'true' },
{ id: 'c3', source_node_id: 'is-pro', target_node_id: 'basic', label: 'false' },
],
})
await widget.embed()
await widget.open()
await expect(widget.lastAgentMessage()).toHaveText('Which plan are you on? (pro/basic)', AI_REPLY)
await widget.send('pro')
await expect(widget.lastAgentMessage()).toHaveText('Pro customers get priority support.', AI_REPLY)
await expect(widget.agentMessages).toContainText(['Thanks — pro it is.'])
await expect(widget.agentMessages.filter({ hasText: 'Basic customers' })).toHaveCount(0)
})
// Plan AND seats, built the way the dashboard's condition builder saves it.
function planAndSeats(wired: Array<'true' | 'false'> = ['true', 'false']): WorkflowGraph {
return {
nodes: [
{ id: 'ask-plan', node_type: 'user_input', name: 'Ask Plan', config: { prompt_message: 'Which plan are you on?' } },
{ id: 'ask-seats', node_type: 'user_input', name: 'Ask Seats', config: { prompt_message: 'How many seats?' } },
{
id: 'big-pro', node_type: 'condition', name: 'Big Pro',
config: {
condition_groups: [{
id: 'g1', logicalOperator: 'AND', rules: [
{ id: 'r1', variable: 'ask_plan_input', operator: '===', value: 'pro' },
{ id: 'r2', variable: 'ask_seats_input', operator: '>', value: '3' },
],
}],
condition_expression: '({{ask_plan_input}} === "pro" AND {{ask_seats_input}} > 3)',
},
},
{ id: 'priority', node_type: 'message', name: 'Priority', config: { message_text: 'You get a dedicated account manager.' } },
{ id: 'standard', node_type: 'message', name: 'Standard', config: { message_text: 'Our team will get back to you.' } },
],
connections: [
{ id: 'c1', source_node_id: 'ask-plan', target_node_id: 'ask-seats' },
{ id: 'c2', source_node_id: 'ask-seats', target_node_id: 'big-pro' },
...wired.map((branch) => ({ id: `c-${branch}`, source_node_id: 'big-pro', target_node_id: branch === 'true' ? 'priority' : 'standard', label: branch })),
],
}
}
const answers = [
{ plan: 'pro', seats: '5', reply: 'You get a dedicated account manager.', why: 'both rules hold' },
{ plan: 'pro', seats: '2', reply: 'Our team will get back to you.', why: 'one rule fails' },
{ plan: 'basic" OR "a" === "a', seats: '5', reply: 'Our team will get back to you.', why: 'an answer that looks like a condition is just text' },
{ plan: 'pro', seats: 'lots', reply: 'Our team will get back to you.', why: 'a non-number is never "greater than"' },
]
for (const { plan, seats, reply, why } of answers) {
test(`a combined condition routes on both answers: ${why} @workflows`, async ({ agent, adminApi, widget }) => {
await publishWorkflow(adminApi, agent.id, planAndSeats())
await widget.embed()
await widget.open()
await expect(widget.lastAgentMessage()).toHaveText('Which plan are you on?', AI_REPLY)
await widget.send(plan)
await expect(widget.lastAgentMessage()).toHaveText('How many seats?', AI_REPLY)
await widget.send(seats)
await expect(widget.lastAgentMessage()).toHaveText(reply, AI_REPLY)
})
}
test('a condition with no branch for its result stops instead of taking the other one @workflows', async ({ agent, adminApi, widget, llm }) => {
await publishWorkflow(adminApi, agent.id, planAndSeats(['true']))
await widget.embed()
await widget.open()
await expect(widget.lastAgentMessage()).toHaveText('Which plan are you on?', AI_REPLY)
await widget.send('basic')
await expect(widget.lastAgentMessage()).toHaveText('How many seats?', AI_REPLY)
await widget.send('1')
await expectUnchanged('agent messages', () => widget.agentMessages.allTextContents())
await expect(widget.agentMessages.filter({ hasText: 'dedicated account manager' })).toHaveCount(0)
expect(await llm.requests(agent)).toHaveLength(0)
})
})
e2e/src/api/agent-socket.ts
+58
View File
@@ -0,0 +1,58 @@
import type { APIRequestContext } from '@playwright/test'
import { env } from '../env'
// The dashboard's live-chat (/agent) socket, driven at the protocol level the
// way another site's page would reach it: with the browser's sign-in cookies
// and an Origin of the page's choosing (or none).
const SOCKET = `${env.backendUrl}/socket.io/?EIO=4`
export type SocketOutcome = 'signed in' | 'refused'
/** The sign-in cookies a browser would attach, as a Cookie header. */
export async function signInCookies(request: APIRequestContext, email: string, password: string): Promise<string> {
const response = await request.post(`${env.apiUrl}/users/login`, { form: { username: email, password } })
if (!response.ok()) throw new Error(`login ${email} → ${response.status()}`)
const { cookies } = await request.storageState()
return cookies.filter((c) => ['access_token', 'refresh_token'].includes(c.name)).map((c) => `${c.name}=${c.value}`).join('; ')
}
/**
* Open /agent over a WebSocket, as the dashboard does. Only the namespace's own
* answer counts: a handshake refused before it (the origin not allowed at all)
* rejects rather than passing as a refusal.
*/
export function agentSocketOverWebSocket(origin: string, cookie: string): Promise<SocketOutcome> {
return new Promise((resolve, reject) => {
// Node's WebSocket takes headers, which a page can't set but a browser sends for it
const ws = new WebSocket(`${SOCKET.replace(/^http/, 'ws')}&transport=websocket`, { headers: { Origin: origin, Cookie: cookie } } as never)
const fail = (why: string) => { clearTimeout(timer); ws.close(); reject(new Error(`${origin}: ${why}`)) }
const timer = setTimeout(() => fail('no answer to the namespace connect'), 10_000)
ws.onmessage = ({ data }) => {
const packet = String(data)
if (packet.startsWith('0')) ws.send('40/agent,')
else if (packet.startsWith('40/agent')) settle('signed in')
else if (packet.startsWith('44/agent')) settle('refused')
}
ws.onerror = () => fail('the socket handshake was refused before the namespace answered')
function settle(outcome: SocketOutcome) { clearTimeout(timer); ws.onerror = null; ws.close(); resolve(outcome) }
})
}
/**
* Open /agent over polling: the handshake with `handshake` headers and `query`
* (a <script> tag sends `j=0`, the cookies and no Origin), then the namespace
* connect and the read from `pageOrigin`. socket.io signs the namespace in with
* the handshake's headers, which is what this exercises.
*/
export async function agentSocketOverPolling(
request: APIRequestContext, cookie: string, pageOrigin: string, handshake: Record<string, string>, query = '',
): Promise<SocketOutcome> {
const opened = await (await request.get(`${SOCKET}&transport=polling${query}`, { headers: { Cookie: cookie, ...handshake } })).text()
const sid = opened.match(/sid\\?":\\?"([^"\\]+)/)?.[1]
if (!sid) throw new Error(`the handshake answered ${opened.slice(0, 80)}`)
const session = `${SOCKET}&transport=polling&sid=${sid}`
await request.post(session, { headers: { Origin: pageOrigin, 'Content-Type': 'text/plain' }, data: '40/agent,' })
const answer = await (await request.get(session, { headers: { Origin: pageOrigin } })).text()
return answer.includes('40/agent,{"sid"') ? 'signed in' : 'refused'
}
e2e/src/api/agents.ts
+47
View File
@@ -0,0 +1,47 @@
import type { ApiClient } from './client'
export interface Agent {
id: string
name: string
}
export interface NewAgent {
name: string
instructions: string[]
agentType?: 'customer_support' | 'sales' | 'tech_support' | 'general' | 'custom'
transferToHuman?: boolean
/** Handover only happens if the agent has at least one group with an online member. */
groupIds?: string[]
/** The widget asks for a star rating when the agent ends the chat. */
askForRating?: boolean
/** The widget only loads with a conversation token from /generate-token. */
requireTokenAuth?: boolean
/** Offer the AI's quick replies as tappable buttons on messaging channels. */
channelQuickReplies?: boolean
/** Visitors may attach files (once a person has joined the chat). */
allowAttachments?: boolean
}
export async function createAgent(api: ApiClient, agent: NewAgent): Promise<Agent> {
const created = await api.post<Agent>('/agent', {
data: {
name: agent.name,
agent_type: agent.agentType ?? 'customer_support',
instructions: agent.instructions,
transfer_to_human: agent.transferToHuman ?? false,
ask_for_rating: agent.askForRating ?? false,
require_token_auth: agent.requireTokenAuth ?? false,
channel_quick_replies: agent.channelQuickReplies ?? false,
allow_attachments: agent.allowAttachments ?? false,
},
})
if (agent.groupIds?.length) {
await api.put(`/agent/${created.id}/groups`, { data: agent.groupIds })
}
return { id: String(created.id), name: created.name }
}
/** Turns lead capture on without the consent gate, so a scripted reply records the lead. */
export async function enableLeadCapture(api: ApiClient, agentId: string): Promise<void> {
await api.put(`/agent/${agentId}/lead-capture`, { data: { enabled: true, require_consent: false } })
}
e2e/src/api/ai.ts
+39
View File
@@ -0,0 +1,39 @@
import { env, seedOrg } from '../env'
import type { ApiClient } from './client'
/**
* Points the org's AI at the fake LLM. The backend live-tests the key on save, so
* the fake must already be running (playwright.config.ts starts it).
*/
export async function useFakeLlm(api: ApiClient): Promise<void> {
await api.post('/ai/setup', {
data: {
model_type: 'OPENAI_COMPATIBLE',
model_name: seedOrg.llmModel,
api_key: 'e2e-fake-key',
base_url: `${env.fakeLlmUrl}/v1`,
},
})
}
export interface AiConfig {
model_type: string
model_name: string
}
export async function currentAiConfig(api: ApiClient): Promise<AiConfig> {
return api.get<AiConfig>('/ai/config')
}
/**
* Puts the org back on the AI it had before a test changed it: the fake LLM, or
* (enterprise) the ChatterMate AI its signup created, which reaches the fake too.
*/
export async function restoreAi(api: ApiClient, before: AiConfig): Promise<void> {
if (before.model_type.toUpperCase() === 'CHATTERMATE') {
// As the dashboard's onboarding sends it: the backend injects the key.
await api.post('/ai/setup', { data: { model_type: 'CHATTERMATE', model_name: 'chattermate', api_key: '' } })
} else {
await useFakeLlm(api)
}
}
e2e/src/api/auth.ts
+25
View File
@@ -0,0 +1,25 @@
import type { ApiClient } from './client'
interface TokenResponse {
access_token: string
user: { id: string; organization_id: string }
}
export interface Session {
token: string
userId: string
organizationId: string
}
/** A client signed in as `email` that signs in again whenever its token expires. */
export async function signedInClient(api: ApiClient, email: string, password: string): Promise<{ api: ApiClient; session: Session }> {
const session = await login(api, email, password)
const renew = async () => (await login(api, email, password)).token
return { api: api.withToken(session.token, renew), session }
}
/** Logs in through the real endpoint; the token is for seeding calls. */
export async function login(api: ApiClient, email: string, password: string): Promise<Session> {
const body = await api.post<TokenResponse>('/users/login', { form: { username: email, password } })
return { token: body.access_token, userId: String(body.user.id), organizationId: String(body.user.organization_id) }
}
e2e/src/api/channels.ts
+72
View File
@@ -0,0 +1,72 @@
import type { ApiClient } from './client'
import type { Schemas } from './types'
/** A connected account (ChannelAccountOut); webhook_url is set for SMS and email. */
export type ChannelAccount = Schemas['ChannelAccountOut']
async function connect(api: ApiClient, path: string, data: Record<string, unknown>): Promise<ChannelAccount> {
const account = await api.post<ChannelAccount>(path, { data })
return { ...account, id: String(account.id) }
}
/** Connects a Telegram bot (the backend checks the token with getMe and registers its webhook). */
export function connectTelegram(api: ApiClient, botToken: string): Promise<ChannelAccount> {
return connect(api, '/channels/telegram', { bot_token: botToken })
}
/** Manual WhatsApp Cloud API connect: a phone number id and a system-user token, plus the
* app secret when the number is in the customer's own Meta app. */
export function connectWhatsApp(api: ApiClient, phoneNumberId: string, accessToken: string, appSecret?: string): Promise<ChannelAccount> {
return connect(api, '/channels/meta/whatsapp', { phone_number_id: phoneNumberId, access_token: accessToken, app_secret: appSecret })
}
export function connectMessenger(api: ApiClient, pageId: string, pageAccessToken: string, appSecret?: string): Promise<ChannelAccount> {
return connect(api, '/channels/meta/messenger', { page_id: pageId, page_access_token: pageAccessToken, app_secret: appSecret })
}
export function connectInstagram(api: ApiClient, igId: string, pageAccessToken: string): Promise<ChannelAccount> {
return connect(api, '/channels/meta/instagram', { ig_id: igId, page_access_token: pageAccessToken })
}
export function connectLine(api: ApiClient, channelSecret: string, channelAccessToken: string): Promise<ChannelAccount> {
return connect(api, '/channels/line', { channel_secret: channelSecret, channel_access_token: channelAccessToken })
}
/** An email inbox that receives mail through the provider's inbound-parse webhook; replies use platform SMTP. */
export function connectEmail(api: ApiClient, inboundAddress: string): Promise<ChannelAccount> {
return connect(api, '/channels/email', { inbound_address: inboundAddress })
}
export function connectSms(
api: ApiClient,
provider: 'twilio' | 'plivo',
phoneNumber: string,
credentials: Record<string, string>,
): Promise<ChannelAccount> {
return connect(api, '/channels/sms', { provider, phone_number: phoneNumber, credentials })
}
/**
* Slack connects only through OAuth: /install hands out a state, Slack sends the
* browser back to /callback with a code, and the backend trades the code for a
* bot token and stores the workspace. The callback answers with a redirect, not
* the account — find it afterwards with listChannelAccounts.
*/
export async function installSlack(api: ApiClient, code: string): Promise<void> {
const install = await api.raw('GET', '/channels/slack/install', { maxRedirects: 0 })
const state = new URL(install.headers().location ?? 'about:blank').searchParams.get('state')
if (!state) throw new Error(`Slack install → ${install.status()}: no state in the redirect`)
const callback = await api.raw('GET', '/channels/slack/callback', { params: { code, state }, maxRedirects: 0 })
const location = callback.headers().location ?? ''
if (!location.includes('status=success')) throw new Error(`Slack callback → ${location || callback.status()}`)
}
export async function listChannelAccounts(api: ApiClient): Promise<ChannelAccount[]> {
const accounts = await api.get<ChannelAccount[]>('/channels/accounts')
return accounts.map((a) => ({ ...a, id: String(a.id) }))
}
/** Routes a connected channel account's conversations to an AI agent. */
export async function routeToAgent(api: ApiClient, accountId: string, agentId: string): Promise<void> {
await api.post(`/channels/agent-config/${accountId}`, { data: { agent_id: agentId, is_active: true } })
}
e2e/src/api/chats.ts
+16
View File
@@ -0,0 +1,16 @@
import type { ApiClient } from './client'
import type { Schemas } from './types'
export type ChatOverview = Schemas['ChatOverviewResponse']
/** The org's conversations with one AI agent, newest first (admin view). */
export async function chatsWithAgent(api: ApiClient, agentId: string): Promise<ChatOverview[]> {
return api.get<ChatOverview[]>('/chats/recent', { params: { agent_id: agentId, limit: 100 } })
}
export type ChatMessage = Schemas['Message']
/** Every stored message in one conversation, oldest first (admin view). */
export async function chatMessages(api: ApiClient, sessionId: string): Promise<ChatMessage[]> {
return (await api.get<Schemas['ChatDetailResponse']>(`/chats/${sessionId}`)).messages
}
e2e/src/api/client.ts
+78
View File
@@ -0,0 +1,78 @@
import type { APIRequestContext, APIResponse } from '@playwright/test'
import { env } from '../env'
type Body = Record<string, unknown> | unknown[]
interface CallOptions {
data?: Body
form?: Record<string, string>
params?: Record<string, string | number | boolean>
/** 0 to read a redirect's Location instead of following it. */
maxRedirects?: number
/** A multipart/form-data body — files as { name, mimeType, buffer }. */
multipart?: Record<string, string | { name: string; mimeType: string; buffer: Buffer }>
/** Extra request headers, e.g. an Origin for a CORS check. */
headers?: Record<string, string>
}
/**
* Thin JSON client for the backend API, used to SEED data — specs assert through
* the UI. Domain modules (agents.ts, widgets.ts, …) build on this, so a route or
* payload change is one edit there and nowhere else.
*/
export class ApiClient {
readonly request: APIRequestContext
private token?: string
private readonly renew?: () => Promise<string>
/** `renew` fetches a fresh token when the current one is refused (it lives 30 minutes). */
constructor(request: APIRequestContext, token?: string, renew?: () => Promise<string>) {
this.request = request
this.token = token
this.renew = renew
}
withToken(token: string, renew?: () => Promise<string>): ApiClient {
return new ApiClient(this.request, token, renew)
}
get<T>(path: string, options: CallOptions = {}) { return this.call<T>('GET', path, options) }
post<T>(path: string, options: CallOptions = {}) { return this.call<T>('POST', path, options) }
put<T>(path: string, options: CallOptions = {}) { return this.call<T>('PUT', path, options) }
patch<T>(path: string, options: CallOptions = {}) { return this.call<T>('PATCH', path, options) }
delete<T>(path: string, options: CallOptions = {}) { return this.call<T>('DELETE', path, options) }
/** Like call(), but hands back the raw response instead of throwing on a non-2xx. */
async raw(method: string, path: string, options: CallOptions = {}): Promise<APIResponse> {
const response = await this.send(method, path, options)
// A long-lived worker outlasts its token; sign in again once and retry.
if (response.status() === 401 && this.renew) {
this.token = await this.renew()
return this.send(method, path, options)
}
return response
}
private send(method: string, path: string, options: CallOptions): Promise<APIResponse> {
return this.request.fetch(`${env.apiUrl}${path}`, {
method,
data: options.data,
form: options.form,
multipart: options.multipart,
params: options.params,
maxRedirects: options.maxRedirects,
headers: { ...(this.token ? { Authorization: `Bearer ${this.token}` } : {}), ...options.headers },
failOnStatusCode: false,
})
}
private async call<T>(method: string, path: string, options: CallOptions): Promise<T> {
const response = await this.raw(method, path, options)
if (!response.ok()) {
// The status and body in the message are what make a seeding failure debuggable.
throw new Error(`${method} ${path} → ${response.status()}: ${(await response.text()).slice(0, 500)}`)
}
const text = await response.text()
return (text ? JSON.parse(text) : undefined) as T
}
}
e2e/src/api/crm.ts
+26
View File
@@ -0,0 +1,26 @@
import type { ApiClient } from './client'
export type CrmProvider = 'hubspot' | 'pipedrive'
/**
* CRMs connect through OAuth only: /install hands out a state, the CRM sends the
* browser back to /callback with a code, and the backend trades it for tokens.
* One connection per org and CRM — an org-wide change (`*.org.spec.ts`).
*/
export async function connectCrm(api: ApiClient, provider: CrmProvider, code: string): Promise<void> {
const install = await api.raw('GET', `/crm/${provider}/install`, { maxRedirects: 0 })
const state = new URL(install.headers().location ?? 'about:blank').searchParams.get('state')
if (!state) throw new Error(`${provider} install → ${install.status()}: ${await install.text()}`)
const callback = await api.raw('GET', `/crm/${provider}/callback`, { params: { code, state }, maxRedirects: 0 })
const location = callback.headers().location ?? ''
if (!location.includes('status=success')) throw new Error(`${provider} callback → ${location || callback.status()}`)
}
export async function disconnectCrm(api: ApiClient, provider: CrmProvider): Promise<void> {
await api.raw('DELETE', `/crm/${provider}`)
}
/** "Sync now" on a person: pushes them to every connected CRM, synchronously. */
export async function syncPersonToCrm(api: ApiClient, personId: string): Promise<{ synced: Array<{ provider: string }> }> {
return api.post(`/people/${personId}/crm-sync`)
}
e2e/src/api/customization.ts
+6
View File
@@ -0,0 +1,6 @@
import type { ApiClient } from './client'
/** The agent's widget look (Agent → Customize), partial updates allowed. */
export async function customizeAgent(api: ApiClient, agentId: string, customization: Record<string, unknown>): Promise<void> {
await api.post(`/agent/${agentId}/customization`, { data: customization })
}
e2e/src/api/help-center.ts
+23
View File
@@ -0,0 +1,23 @@
import { env } from '../env'
import type { ApiClient } from './client'
export interface Article {
id: string
slug: string | null
status: 'draft' | 'published'
}
/** Turns the org's public help center on (idempotent) and returns its public base URL. */
export async function enableHelpCenter(api: ApiClient): Promise<string> {
const settings = await api.put<{ slug: string }>('/help-center/settings', { data: { enabled: true } })
// .env.e2e serves help centers by path (HELP_CENTER_PUBLIC_MODE=path), as self-hosters do.
return `${env.backendUrl}/help/${settings.slug}`
}
export async function createArticle(api: ApiClient, article: { question: string; answer: string }): Promise<Article> {
return api.post<Article>('/help-center/faqs', { data: { ...article, status: 'draft' } })
}
export async function setArticleStatus(api: ApiClient, id: string, status: Article['status']): Promise<Article> {
return api.put<Article>(`/help-center/faqs/${id}`, { data: { status } })
}
e2e/src/api/integrations.ts
+70
View File
@@ -0,0 +1,70 @@
import { createHmac } from 'node:crypto'
import type { ApiClient } from './client'
import { env } from '../env'
// Jira and Shopify connect once per org (`*.org.spec.ts`); both reach the recording
// fake through JIRA_*_BASE / SHOPIFY_ADMIN_BASE in .env.e2e.
/** The project and issue type the fake Jira site has (services/fake-externals). */
export const fakeJira = { projectKey: 'SUP', taskTypeId: '10001' } as const
/**
* Connects Jira through its OAuth callback: /authorize hands out a state, Atlassian
* sends the browser back to /callback with a code, and the backend trades it for a
* token and the site (the fake derives both from the code).
*/
export async function connectJira(api: ApiClient, code: string): Promise<void> {
const authorize = await api.raw('GET', '/jira/authorize', { maxRedirects: 0 })
const state = new URL(authorize.headers().location ?? 'about:blank').searchParams.get('state')
if (!state) throw new Error(`Jira authorize → ${authorize.status()}: ${await authorize.text()}`)
const callback = await api.raw('GET', '/jira/callback', { params: { code, state }, maxRedirects: 0 })
const location = callback.headers().location ?? ''
if (!location.includes('status=success')) throw new Error(`Jira callback → ${location || callback.status()}`)
}
export async function disconnectJira(api: ApiClient): Promise<void> {
await api.raw('DELETE', '/jira/disconnect')
}
/** The agent files Jira issues into the fake site's Support project, as Tasks. */
export async function routeTicketsToJira(api: ApiClient, agentId: string): Promise<void> {
await api.post(`/jira/agent-config/${agentId}`, { data: { enabled: true, projectKey: fakeJira.projectKey, issueTypeId: fakeJira.taskTypeId } })
}
/** An HS256 session token, as Shopify's admin hands the embedded app. */
function shopifySessionToken(shopDomain: string): string {
const encode = (value: object) => Buffer.from(JSON.stringify(value)).toString('base64url')
const now = Math.floor(Date.now() / 1000)
const header = encode({ alg: 'HS256', typ: 'JWT' })
const claims = encode({
iss: `https://${shopDomain}/admin`, dest: `https://${shopDomain}`, aud: env.shopifyApp.key,
sub: '1', exp: now + 60, nbf: now - 5, iat: now, jti: `${shopDomain}-${now}`,
})
const signature = createHmac('sha256', env.shopifyApp.secret).update(`${header}.${claims}`).digest('base64url')
return `${header}.${claims}.${signature}`
}
/**
* Installs the app on `shopDomain` (*.myshopify.com) and links the shop to the
* admin's org, as the embedded app does: the session token is traded for an
* offline token, then the signed-in admin claims the shop. Linking also sets the
* org's domain to the shop's — restore it with `uninstallShopify`.
*/
export async function installShopify(api: ApiClient, shopDomain: string): Promise<string> {
const { shop_id: shopId } = await api.post<{ shop_id: string }>('/shopify/exchange-token', {
// The session token stands in for the admin's: this is the embedded app calling.
headers: { Authorization: `Bearer ${shopifySessionToken(shopDomain)}` },
})
await api.post('/shopify/link-organization', { data: { shop_id: shopId } })
return shopId
}
export async function uninstallShopify(api: ApiClient, shopId: string, orgId: string, orgDomain: string): Promise<void> {
await api.raw('DELETE', `/shopify/shops/${shopId}`)
await api.patch(`/organizations/${orgId}`, { data: { domain: orgDomain } })
}
/** The agent answers order questions from this shop. */
export async function useShopForAgent(api: ApiClient, agentId: string, shopId: string): Promise<void> {
await api.post(`/shopify/agent-config/${agentId}`, { data: { enabled: true, shop_id: shopId } })
}
e2e/src/api/knowledge.ts
+47
View File
@@ -0,0 +1,47 @@
import type { ApiClient } from './client'
/**
* Pasted-text knowledge is indexed synchronously, so it is searchable as soon as
* this returns. Returns the source's id.
*/
export async function addKnowledgeText(
api: ApiClient,
source: { orgId: string; agentId: string; title: string; content: string },
): Promise<number> {
const body = await api.post<{ knowledge: { id: number } }>('/knowledge/add/text', {
data: { org_id: source.orgId, agent_id: source.agentId, title: source.title, content: source.content },
})
return body.knowledge.id
}
/** Uploads a PDF to the knowledge base. It is queued; the knowledge worker indexes it. */
export async function uploadKnowledgePdf(
api: ApiClient,
source: { orgId: string; agentId: string; filename: string; pdf: Buffer },
): Promise<void> {
await api.post('/knowledge/upload/pdf', {
multipart: {
org_id: source.orgId,
agent_id: source.agentId,
files: { name: source.filename, mimeType: 'application/pdf', buffer: source.pdf },
},
})
}
/**
* The agent's sources the knowledge worker has not finished: pending or still
* processing, and any that failed (with why). Empty once everything is indexed.
*/
export async function unfinishedKnowledge(
api: ApiClient,
agentId: string,
): Promise<{ source: string; status: string; error: string | null }[]> {
const body = await api.get<{ queue_items: { source: string; status: string; error: string | null }[] }>(
`/knowledge/queue/agent/${agentId}`,
)
return body.queue_items.map(({ source, status, error }) => ({ source, status, error }))
}
export async function deleteKnowledge(api: ApiClient, knowledgeId: number): Promise<void> {
await api.delete(`/knowledge/${knowledgeId}`)
}
e2e/src/api/mcp.ts
+18
View File
@@ -0,0 +1,18 @@
import type { ApiClient } from './client'
export interface McpConnector {
id: number
name: string
url: string | null
headers: Record<string, string> | null
}
/** Adds a streamable-HTTP MCP server to the org's connectors. */
export async function createMcpConnector(api: ApiClient, connector: { name: string; url: string; headers?: Record<string, string> }): Promise<McpConnector> {
return api.post<McpConnector>('/mcp-tools', { data: { transport_type: 'http', enabled: true, timeout: 10, ...connector } })
}
/** Gives an agent the connector's tools. */
export async function attachMcpConnector(api: ApiClient, connectorId: number, agentId: string): Promise<void> {
await api.post('/mcp-tools/agent-association', { data: { mcp_tool_id: connectorId, agent_id: agentId } })
}
e2e/src/api/organizations.ts
+36
View File
@@ -0,0 +1,36 @@
import type { ApiClient } from './client'
const DAYS = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] as const
export async function isSetUp(api: ApiClient): Promise<boolean> {
const body = await api.get<{ is_setup: boolean }>('/organizations/setup-status')
return body.is_setup
}
/**
* Open every day, all day. Handover only reaches a human inside business hours, so
* without this a nightly run outside 9–5 would take the "we'll email you" branch.
*/
export async function setAlwaysOpen(api: ApiClient, orgId: string): Promise<void> {
await setBusinessHours(api, orgId, true)
}
/** Closed every day — for after-hours journeys (an org-wide change: `*.org.spec.ts` only). */
export async function setAlwaysClosed(api: ApiClient, orgId: string): Promise<void> {
await setBusinessHours(api, orgId, false)
}
async function setBusinessHours(api: ApiClient, orgId: string, open: boolean): Promise<void> {
const businessHours = Object.fromEntries(DAYS.map((day) => [day, { start: '00:00', end: '23:59', enabled: open }]))
await api.patch(`/organizations/${orgId}`, { data: { business_hours: businessHours } })
}
/** The org's website, to put back after a spec changes it. */
export async function orgDomain(api: ApiClient, orgId: string): Promise<string> {
return (await api.get<{ domain: string }>(`/organizations/${orgId}`)).domain
}
/** Change the org's website, which decides the sites allowed to call the API (`*.org.spec.ts` only). */
export async function setOrgDomain(api: ApiClient, orgId: string, domain: string): Promise<void> {
await api.patch(`/organizations/${orgId}`, { data: { domain } })
}
e2e/src/api/people.ts
+16
View File
@@ -0,0 +1,16 @@
import type { ApiClient } from './client'
export interface Person {
id: string
name: string | null
email: string | null
phone: string | null
lead_stage: 'visitor' | 'lead' | 'customer'
}
/** The one person whose email or phone matches `query` (People search). */
export async function findPerson(api: ApiClient, query: string): Promise<Person> {
const { items } = await api.get<{ items: Person[] }>('/people', { params: { search: query } })
if (items.length !== 1) throw new Error(`expected one person matching ${query}, found ${items.length}`)
return api.get<Person>(`/people/${items[0].id}`)
}
e2e/src/api/schema.d.ts
Vendored
+18324
View File
File diff suppressed because it is too large. Load diff
e2e/src/api/team.ts
+72
View File
@@ -0,0 +1,72 @@
import type { ApiClient } from './client'
export interface User {
id: string
email: string
full_name: string
}
interface Role {
id: number
name: string
}
interface Group {
id: string
name: string
users?: User[]
}
export async function roleId(api: ApiClient, name: string): Promise<number> {
const roles = await api.get<Role[]>('/roles')
const role = roles.find((r) => r.name === name)
if (!role) throw new Error(`no role named '${name}' (have: ${roles.map((r) => r.name).join(', ')})`)
return role.id
}
export interface NewUser {
email: string
name: string
password: string
role: string
/** The user form's chat-scope toggles; omitted = the role's own scope. */
seeAllAiChats?: boolean
seeAllOrgChats?: boolean
}
export async function createUser(api: ApiClient, user: NewUser): Promise<User> {
const created = await api.post<User>('/users', {
data: {
email: user.email, full_name: user.name, password: user.password, role_id: await roleId(api, user.role),
see_all_ai_chats: user.seeAllAiChats, see_all_org_chats: user.seeAllOrgChats,
},
})
return { ...created, id: String(created.id) }
}
/** A new group with `members`. */
export async function createGroup(api: ApiClient, name: string, members: User[]): Promise<string> {
const group = await api.post<Group>('/groups', { data: { name } })
for (const member of members) await api.post(`/groups/${group.id}/users/${member.id}`)
return String(group.id)
}
/** Creates the user unless one with that email exists — setup reruns against a reused DB. */
export async function ensureUser(
api: ApiClient,
user: NewUser,
): Promise<User> {
const existing = (await api.get<User[]>('/users')).find((u) => u.email === user.email)
if (existing) return { ...existing, id: String(existing.id) }
return createUser(api, user)
}
export async function ensureGroup(api: ApiClient, name: string, members: User[]): Promise<string> {
const groups = await api.get<Group[]>('/groups')
const group = groups.find((g) => g.name === name) ?? (await api.post<Group>('/groups', { data: { name } }))
const memberIds = new Set((group.users ?? []).map((u) => String(u.id)))
for (const member of members) {
if (!memberIds.has(member.id)) await api.post(`/groups/${group.id}/users/${member.id}`)
}
return String(group.id)
}
e2e/src/api/tickets.ts
+9
View File
@@ -0,0 +1,9 @@
import type { ApiClient } from './client'
/**
* New tickets are queued for the AI investigator by default; the e2e stack runs no
* worker, so setup turns that off and tickets stay exactly as the test made them.
*/
export async function disableAutoInvestigation(api: ApiClient): Promise<void> {
await api.put('/tickets/settings', { data: { auto_investigate_on_create: false } })
}
e2e/src/api/types.ts
+8
View File
@@ -0,0 +1,8 @@
import type { components } from './schema'
/**
* Request and response shapes straight from the backend's OpenAPI schema
* (regenerate with scripts/api-types.sh; CI fails when it is stale). Use these
* instead of hand-written interfaces, so a renamed field is a type error here.
*/
export type Schemas = components['schemas']
Loaded 100 of 216 files, more files were not shown because too many files have changed in this diff. Show more