From f967f55504d30ecd27fe1e263d885e0385015e5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:42:08 -0700 Subject: [PATCH 01/83] docs: design spec for sync mapping persistence + bulk force-sync Two stubbed endpoints behind the already-built /sync dashboard (mapping config PUT, bulk force-sync) never got finished. Specs closing both using existing SystemConfig table and TRACKER_SYNC job - no new schema or job types needed. --- ...-persistence-and-bulk-force-sync-design.md | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md diff --git a/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md new file mode 100644 index 00000000..71d6c9be --- /dev/null +++ b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md @@ -0,0 +1,113 @@ +# Sync Mapping Persistence + Bulk Force-Sync + +## Context + +The Bidirectional Sync feature (GitHub ↔ Linear ↔ Outpost) is already built: `SyncEngine`, +plugin adapters, `TRACKER_SYNC` job handler wired into `apps/worker`, triggers, echo +detection, and a `/sync` dashboard. Two endpoints behind that dashboard are still 501 stubs: + +- `PUT /api/sync/mappings` (`apps/web/src/app/api/sync/mappings/route.ts:111`) — validates + the body but never persists it. `GET` always returns hardcoded `DEFAULT_*` constants. +- `POST /api/sync/force` (`apps/web/src/app/api/sync/force/route.ts:43`) — only handles a + single ticket's sync (currently unimplemented for that path too); has no bulk/full mode. + +This spec closes both gaps without introducing new architecture — reusing the existing +`SystemConfig` key-value table and the existing `TRACKER_SYNC` job/handler. + +## Goals + +1. Editing status/priority/label mappings in the `/sync` dashboard actually persists, and + actually changes what the running sync worker does — not just a UI-only save. +2. An admin can trigger a full resync of every ticket linked to a given plugin (Linear or + GitHub) from the dashboard, without a new job type. + +## Non-goals + +- No new job type. Bulk sync reuses `TRACKER_SYNC` per linked ticket. +- No mapping-config versioning/audit trail — a single current config, overwritten on save. +- No UI changes — `apps/web/src/components/sync/*` already renders/edits this data; only + the API routes and the worker's config loading change. + +## Design + +### 1. Mapping persistence + +Store the full mapping config as one JSON row in the existing `SystemConfig` model +(`packages/outpost/db/prisma/schema.prisma:430-434`, no migration needed): + +- `key`: `"sync.mappingConfig"` +- `value`: JSON string of `{ statusMappings, priorityMappings, labelRules }` (same shape the + route already validates) + +**`GET /api/sync/mappings`**: read the `SystemConfig` row. If present, parse and return it +(merged with identity mappings from `ExternalIdentity`, as today). If absent, fall back to +the current `DEFAULT_STATUS_MAPPINGS` / `DEFAULT_PRIORITY_MAPPINGS` / `DEFAULT_LABEL_RULES` +constants — unchanged behavior for a fresh install. + +**`PUT /api/sync/mappings`**: after existing validation, `prisma.systemConfig.upsert()` the +row. Return the saved config. Remove the 501. + +**Wiring into the running sync engine** (the part that makes this real, not cosmetic): +today `createLinearStatusMap()` / `createGitHubStatusMap()` in +`packages/outpost/shared/src/sync/status-map.ts:69-86` are hardcoded factories, called once +at worker boot in `apps/worker/src/index.ts`. Add: + +```ts +// status-map.ts +export async function loadStatusMap(plugin: 'linear' | 'github'): Promise +``` + +which reads `SystemConfig["sync.mappingConfig"]`, extracts `statusMappings[plugin]` if +present, and builds a `StatusMap` from it; falls back to `createLinearStatusMap()` / +`createGitHubStatusMap()` if the config row or plugin key is missing. `apps/worker/src/index.ts` +calls this at startup instead of the hardcoded factories directly. + +Priority mappings and label rules are read/written the same way but are not yet consumed +elsewhere in the sync engine — persisting them is still correct (single source of truth, +dashboard round-trips real data) but wiring them into `executePush`'s priority/label +handling is out of scope here (they're already applied as literal values passed by the +caller, not looked up from a plugin-level map, per `tracker-sync.ts:181-188`). + +### 2. Bulk force-sync + +`POST /api/sync/force` body: `{ plugin: string, ticketId?: string }`. + +- `ticketId` present → sync that one ticket (existing single-ticket path; still needs the + same "current values" push described below, since it's currently also unimplemented). +- `ticketId` absent → bulk mode: + 1. `prisma.ticketExternalLink.findMany({ where: { plugin }, include: { ticket: true } })` + 2. For each linked ticket, enqueue three `TRACKER_SYNC` jobs (reusing the existing job + type/handler, untouched) via `createJob`: + - `action: 'status_change'`, `changeData: { status: ticket.status }` + - `action: 'priority_change'`, `changeData: { priority: ticket.priority }` + - `action: 'label_change'`, `changeData: { labels: ticket.tags }` (skip if empty) + 3. Return `{ queued: , jobs: }`. + +The route only inserts jobs (cheap Postgres writes); the worker performs the actual pushes +asynchronously, so this stays fast even for a few hundred linked tickets. No pagination +needed at current expected volume; if a workspace ever has thousands of linked tickets, +that's a future problem, not one to solve speculatively here. + +Unknown-plugin handling (`404` if no `SyncEvent` mentions the plugin) is unchanged. + +## Testing + +Per repo convention (Vitest, red-green, webhook/job tests use mocked Prisma): + +- `mappings/route.ts`: test GET falls back to defaults when no `SystemConfig` row exists; + test PUT upserts and GET reflects the saved value; test PUT validation still rejects + missing `statusMappings`/`priorityMappings`. +- `status-map.ts` `loadStatusMap`: test it builds from a mocked `SystemConfig` row; test it + falls back to the hardcoded factory when the row or plugin key is absent. +- `force/route.ts`: test bulk mode enqueues 3 jobs per linked ticket (mock + `ticketExternalLink.findMany` returning N tickets, assert `createJob` called 3N times); + test single-ticket mode still works; test unknown plugin still 404s; test empty `tags` + skips the label job. + +## Files touched + +- `apps/web/src/app/api/sync/mappings/route.ts` +- `apps/web/src/app/api/sync/force/route.ts` +- `packages/outpost/shared/src/sync/status-map.ts` +- `apps/worker/src/index.ts` +- New/updated test files alongside each of the above per existing `__tests__/` convention From c5e5dca42e9634a70e321778fb0d8d4afa0702c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:47:57 -0700 Subject: [PATCH 02/83] docs: implementation plan for sync mapping persistence + bulk force-sync 5 tasks, each red-green tested: SystemConfig-backed mapping persistence, loadStatusMap wiring, initializeSyncEngine override support, worker adapter registration fix (Linear was never wired up), bulk force-sync. --- ...mapping-persistence-and-bulk-force-sync.md | 911 ++++++++++++++++++ 1 file changed, 911 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md diff --git a/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md b/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md new file mode 100644 index 00000000..2757bac3 --- /dev/null +++ b/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md @@ -0,0 +1,911 @@ +# Sync Mapping Persistence + Bulk Force-Sync Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finish two 501-stub endpoints behind the `/sync` dashboard (mapping config persistence, bulk force-sync), and make the mapping config actually change the running sync worker's behavior — not just round-trip through the dashboard. + +**Architecture:** Reuse the existing `SystemConfig` key-value table for mapping persistence (no schema change). Reuse the existing `TRACKER_SYNC` job/handler for bulk sync (no new job type). Fix a latent bug found while tracing this: `apps/worker/src/index.ts` builds a bare `SyncEngine` with zero registered plugins — `initializeSyncEngine()` (which registers the Linear adapter) is never called anywhere in `apps/`. Wire that in for Linear only; GitHub adapter registration needs new Octokit-in-worker plumbing and is explicitly out of scope for this plan. + +**Tech Stack:** Next.js API routes, Prisma, Vitest (mocked Prisma per repo convention), the existing `@copilotkit/outpost/shared` sync package. + +## Global Constraints + +- No new Prisma models or migrations — `SystemConfig` (key/value) already exists. +- No new `JobType` — bulk sync reuses `TRACKER_SYNC` per linked ticket. +- `Ticket` has no tags/labels field in the current schema — bulk resync only pushes `status_change` and `priority_change`, never `label_change`. +- GitHub adapter registration in the worker is out of scope — only the Linear adapter gets wired up. Bulk force-sync for `plugin: 'github'` will still fail with "Plugin not registered", same as before this plan (not a regression). +- Every task ends with passing tests using this repo's existing mocked-Prisma Vitest convention (see `apps/web/src/__tests__/sync-api.test.ts` and `packages/outpost/shared/src/dispatch/__tests__/dispatch.test.ts` for the pattern). No task is done without a red-then-green test cycle. + +--- + +## Task 1: Persist mapping config via SystemConfig + +**Files:** +- Modify: `apps/web/src/app/api/sync/mappings/route.ts` +- Modify: `apps/web/src/__tests__/sync-api.test.ts:229-272` (the two `describe` blocks for `GET`/`PUT /api/sync/mappings`) + +**Interfaces:** +- Consumes: `prisma.systemConfig.findUnique({ where: { key } })` / `.upsert({ where, update, create })` — same shape already used in `packages/outpost/shared/src/dispatch/on-call.ts:45-60`. +- Produces: `MAPPING_CONFIG_KEY = 'sync.mappingConfig'` constant (exported from this route file) — Task 2 imports the same string literal into `status-map.ts` (kept as a plain string constant, not cross-imported, to avoid a web→shared reverse dependency; both sides must use the exact string `'sync.mappingConfig'`). + +- [ ] **Step 1: Write the failing tests** + +Replace the two existing `describe` blocks at the bottom of `apps/web/src/__tests__/sync-api.test.ts` (lines 229-272) with: + +```typescript +describe('GET /api/sync/mappings', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetServerSession.mockResolvedValue(userSession('tm-1')); + }); + + it('falls back to defaults when no SystemConfig row exists', async () => { + mockExternalIdentityFindMany.mockResolvedValue([]); + mockSystemConfigFindUnique.mockResolvedValue(null); + + const res = await getMappings(); + const body = await res.json(); + + expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); + expect(body.priorityMappings).toBeDefined(); + expect(body.identityMappings).toBeDefined(); + expect(body.labelRules).toBeDefined(); + }); + + it('returns the persisted config when a SystemConfig row exists', async () => { + mockExternalIdentityFindMany.mockResolvedValue([]); + const saved = { + statusMappings: { linear: [{ externalStatus: 'Custom', outpostStatus: 'OPEN' }] }, + priorityMappings: { linear: [] }, + labelRules: { linear: [] }, + }; + mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(saved) }); + + const res = await getMappings(); + const body = await res.json(); + + expect(body.statusMappings).toEqual(saved.statusMappings); + }); +}); + +describe('PUT /api/sync/mappings', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'ADMIN')); + }); + + it('persists a valid mapping update', async () => { + const config = { + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { linear: [] }, + }; + mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); + const res = await putMappings(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(mockSystemConfigUpsert).toHaveBeenCalledWith({ + where: { key: 'sync.mappingConfig' }, + update: { value: JSON.stringify(config) }, + create: { key: 'sync.mappingConfig', value: JSON.stringify(config) }, + }); + expect(body.statusMappings).toEqual(config.statusMappings); + }); + + it('rejects when required fields missing', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', {}, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + + it('requires admin role', async () => { + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); + + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(403); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); +}); +``` + +Also add the two new mock functions near the top of the file, next to the existing `mockExternalIdentityFindMany` declaration (around line 10) and inside the `vi.mock('@copilotkit/outpost/db', ...)` block (around line 12-25): + +```typescript +const mockSystemConfigFindUnique = vi.fn(); +const mockSystemConfigUpsert = vi.fn(); +``` + +Add `systemConfig` alongside the existing `syncEvent`/`externalIdentity` keys inside the `prisma` mock object: + +```typescript + systemConfig: { + findUnique: (...args: unknown[]) => mockSystemConfigFindUnique(...args), + upsert: (...args: unknown[]) => mockSystemConfigUpsert(...args), + }, +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd apps/web && pnpm test -- sync-api.test.ts` +Expected: FAIL — `GET /api/sync/mappings` tests fail because the route ignores `mockSystemConfigFindUnique` and still returns hardcoded defaults only; `PUT` tests fail with 501 (route still returns "not yet implemented"). + +- [ ] **Step 3: Implement mapping persistence** + +Replace the `GET` and `PUT` handlers in `apps/web/src/app/api/sync/mappings/route.ts` (keep the `DEFAULT_STATUS_MAPPINGS` / `DEFAULT_PRIORITY_MAPPINGS` / `DEFAULT_LABEL_RULES` constants as-is): + +```typescript +export const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; + +interface PersistedMappingConfig { + statusMappings: typeof DEFAULT_STATUS_MAPPINGS; + priorityMappings: typeof DEFAULT_PRIORITY_MAPPINGS; + labelRules?: typeof DEFAULT_LABEL_RULES; +} + +async function readPersistedConfig(): Promise { + const row = await prisma.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return null; + try { + return JSON.parse(row.value) as PersistedMappingConfig; + } catch { + return null; + } +} + +/** + * GET /api/sync/mappings + * + * Returns the current mapping configuration. Identity mappings are always + * fetched live from the ExternalIdentity table. Status/priority/label + * mappings come from the persisted SystemConfig row if one exists, else + * the code defaults. + */ +export async function GET() { + const { error } = await requireSession(); + if (error) return error; + + const identities = await prisma.externalIdentity.findMany({ + include: { member: { select: { id: true, name: true } } }, + }); + + const identityMappings = identities.map((ei: typeof identities[number]) => ({ + id: ei.id, + externalPlugin: ei.plugin, + externalUserId: ei.externalId, + externalDisplayName: ei.externalId, + memberId: ei.member?.id ?? null, + memberName: ei.member?.name ?? null, + })); + + const persisted = await readPersistedConfig(); + + return NextResponse.json({ + statusMappings: persisted?.statusMappings ?? DEFAULT_STATUS_MAPPINGS, + priorityMappings: persisted?.priorityMappings ?? DEFAULT_PRIORITY_MAPPINGS, + identityMappings, + labelRules: persisted?.labelRules ?? DEFAULT_LABEL_RULES, + }); +} + +/** + * PUT /api/sync/mappings + * + * Persists the mapping configuration as a single JSON row in SystemConfig. + * Body: MappingConfig (statusMappings, priorityMappings required; labelRules optional) + */ +export async function PUT(request: NextRequest) { + const { error } = await requireAdmin(); + if (error) return error; + + try { + const body = await request.json(); + + if (!body.statusMappings || !body.priorityMappings) { + return NextResponse.json( + { error: 'statusMappings and priorityMappings are required' }, + { status: 400 }, + ); + } + + const config: PersistedMappingConfig = { + statusMappings: body.statusMappings, + priorityMappings: body.priorityMappings, + labelRules: body.labelRules ?? DEFAULT_LABEL_RULES, + }; + const value = JSON.stringify(config); + + await prisma.systemConfig.upsert({ + where: { key: MAPPING_CONFIG_KEY }, + update: { value }, + create: { key: MAPPING_CONFIG_KEY, value }, + }); + + return NextResponse.json(config); + } catch { + return NextResponse.json( + { error: 'Invalid request body' }, + { status: 400 }, + ); + } +} +``` + +Note the `requireAdmin` 403 case (test "requires admin role") already works via the existing `requireAdmin()` call — no extra code needed there, it's covered by the auth helper. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd apps/web && pnpm test -- sync-api.test.ts` +Expected: PASS — all `GET`/`PUT /api/sync/mappings` tests green, plus all pre-existing tests in the file still green. + +- [ ] **Step 5: Commit** + +```bash +git add apps/web/src/app/api/sync/mappings/route.ts apps/web/src/__tests__/sync-api.test.ts +git commit -m "feat(sync): persist mapping config to SystemConfig" +``` + +--- + +## Task 2: Load status map from persisted config + +**Files:** +- Modify: `packages/outpost/shared/src/sync/status-map.ts` +- Modify: `packages/outpost/shared/src/sync/index.ts:4` (export the new function) +- Create: `packages/outpost/shared/src/sync/__tests__/status-map.test.ts` + +**Interfaces:** +- Consumes: nothing new from earlier tasks (the `'sync.mappingConfig'` key string must match Task 1's `MAPPING_CONFIG_KEY` value exactly). +- Produces: `loadStatusMap(plugin: 'linear' | 'github', db: StatusMapDb): Promise` and `export interface StatusMapDb`. Task 3 imports both. + +- [ ] **Step 1: Write the failing test** + +Create `packages/outpost/shared/src/sync/__tests__/status-map.test.ts`: + +```typescript +import { describe, it, expect, vi } from 'vitest'; +import { loadStatusMap } from '../status-map.js'; +import { TicketStatus } from '../../types.js'; + +function makeDb(row: { key: string; value: string } | null) { + return { systemConfig: { findUnique: vi.fn().mockResolvedValue(row) } }; +} + +describe('loadStatusMap', () => { + it('falls back to the hardcoded Linear map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + }); + + it('falls back to the hardcoded GitHub map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadStatusMap('github', db); + + expect(map.toOutpost('closed')).toBe(TicketStatus.CLOSED); + }); + + it('builds from persisted config when present for the requested plugin', async () => { + const config = { + statusMappings: { + linear: [{ externalStatus: 'Shipped', outpostStatus: 'RESOLVED' }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Shipped')).toBe(TicketStatus.RESOLVED); + // 'Done' is no longer in the map since the persisted config replaced it entirely + expect(map.toOutpost('Done')).toBe(TicketStatus.OPEN); // StatusMap.toOutpost default fallback + }); + + it('falls back to defaults when persisted config has no entry for this plugin', async () => { + const config = { statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] } }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('github', db); + + expect(map.toOutpost('closed')).toBe(TicketStatus.CLOSED); + }); + + it('falls back to defaults when the persisted value is malformed JSON', async () => { + const db = makeDb({ key: 'sync.mappingConfig', value: 'not json' }); + + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd packages/outpost/shared && pnpm test -- status-map.test.ts` +Expected: FAIL with `loadStatusMap is not a function` / import error (not yet exported). + +- [ ] **Step 3: Implement `loadStatusMap`** + +Append to `packages/outpost/shared/src/sync/status-map.ts` (keep everything already in the file, add below `createLinearStatusMap`): + +```typescript +// ─── Persisted Config Loading ───────────────────────────────────────────── + +/** Must match the key used by apps/web/src/app/api/sync/mappings/route.ts. */ +const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; + +/** Minimal Prisma subset needed to load a persisted mapping config. */ +export interface StatusMapDb { + systemConfig: { + findUnique(args: { where: { key: string } }): Promise<{ key: string; value: string } | null>; + }; +} + +interface PersistedStatusMappingEntry { + externalStatus: string; + outpostStatus: TicketStatus; +} + +/** + * Build a StatusMap for `plugin`, preferring the persisted SystemConfig + * row (written by the /api/sync/mappings dashboard) over the hardcoded + * factory defaults. Falls back to the hardcoded default whenever the + * config row is missing, malformed, or has no entry for this plugin. + */ +export async function loadStatusMap( + plugin: 'linear' | 'github', + db: StatusMapDb, +): Promise { + const fallback = plugin === 'linear' ? createLinearStatusMap() : createGitHubStatusMap(); + + const row = await db.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return fallback; + + let parsed: unknown; + try { + parsed = JSON.parse(row.value); + } catch { + return fallback; + } + + const entries = (parsed as { statusMappings?: Record }) + ?.statusMappings?.[plugin]; + if (!Array.isArray(entries) || entries.length === 0) return fallback; + + const config: StatusMappingConfig = {}; + for (const entry of entries) { + if (entry?.externalStatus && entry?.outpostStatus) { + config[entry.externalStatus] = entry.outpostStatus; + } + } + return Object.keys(config).length > 0 ? new StatusMap(config) : fallback; +} +``` + +Add the export to `packages/outpost/shared/src/sync/index.ts:4`: + +```typescript +export { StatusMap, createGitHubStatusMap, createLinearStatusMap, loadStatusMap } from './status-map.js'; +export type { StatusMappingConfig, StatusMapDb } from './status-map.js'; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd packages/outpost/shared && pnpm test -- status-map.test.ts` +Expected: PASS, all 5 tests green. + +- [ ] **Step 5: Commit** + +```bash +git add packages/outpost/shared/src/sync/status-map.ts packages/outpost/shared/src/sync/index.ts packages/outpost/shared/src/sync/__tests__/status-map.test.ts +git commit -m "feat(sync): load status map from persisted mapping config" +``` + +--- + +## Task 3: Let `initializeSyncEngine` accept a pre-loaded status map + +**Files:** +- Modify: `packages/outpost/shared/src/sync/init.ts` +- Create: `packages/outpost/shared/src/sync/__tests__/init.test.ts` + +**Interfaces:** +- Consumes: `loadStatusMap`, `StatusMapDb` from Task 2 (imported by the caller, not by `init.ts` itself — `init.ts` just accepts an already-built `StatusMap`). +- Produces: `InitOptions.statusMapOverride?: StatusMap` — Task 4's `buildSyncEngine` passes this in. + +- [ ] **Step 1: Write the failing test** + +Create `packages/outpost/shared/src/sync/__tests__/init.test.ts`: + +```typescript +import { describe, it, expect, vi } from 'vitest'; +import { initializeSyncEngine } from '../init.js'; +import { StatusMap } from '../status-map.js'; +import { TicketStatus } from '../../types.js'; + +function makeIdentityDeps() { + return { + externalIdentity: { + findUnique: vi.fn(), + findFirst: vi.fn(), + findMany: vi.fn(), + create: vi.fn(), + }, + }; +} + +describe('initializeSyncEngine', () => { + it('registers no adapters when Linear env vars are missing', () => { + const engine = initializeSyncEngine({ + deps: { prisma: {} as never, createJob: vi.fn() }, + identityDeps: makeIdentityDeps(), + env: {}, + }); + + expect(engine.getPlugin('linear')).toBeUndefined(); + }); + + it('registers the Linear adapter with the default status map when no override is given', () => { + const engine = initializeSyncEngine({ + deps: { prisma: {} as never, createJob: vi.fn() }, + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + }); + + expect(engine.getPlugin('linear')).toBeDefined(); + }); + + it('uses the provided statusMapOverride instead of the hardcoded default', () => { + const customMap = new StatusMap({ Custom: TicketStatus.WAITING_ON_TEAM }); + + const engine = initializeSyncEngine({ + deps: { prisma: {} as never, createJob: vi.fn() }, + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + statusMapOverride: customMap, + }); + + const plugin = engine.getPlugin('linear'); + expect(plugin).toBeDefined(); + // mapStatusToOutpost is the InternalTracker interface method the adapter + // delegates to its injected StatusMap + expect(plugin!.mapStatusToOutpost('Custom')).toBe(TicketStatus.WAITING_ON_TEAM); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd packages/outpost/shared && pnpm test -- init.test.ts` +Expected: FAIL on the third test — `statusMapOverride` is not a recognized option, so the Linear adapter uses `createLinearStatusMap()` regardless, and `plugin.mapStatusToOutpost('Custom')` returns the fallback (`OPEN`), not `WAITING_ON_TEAM`. + +- [ ] **Step 3: Implement `statusMapOverride`** + +Modify `packages/outpost/shared/src/sync/init.ts`: + +```typescript +interface InitOptions { + /** Override for dependency injection (testing). */ + deps: SyncEngineDeps; + /** Override for identity mapper deps (testing). */ + identityDeps?: IdentityMapperDeps; + /** Override for environment variables (testing). */ + env?: Record; + /** Pre-built StatusMap to use instead of createLinearStatusMap(). */ + statusMapOverride?: StatusMap; +} +``` + +And in the Linear adapter construction, replace `statusMap: createLinearStatusMap(),` with: + +```typescript + statusMap: options.statusMapOverride ?? createLinearStatusMap(), +``` + +Add `StatusMap` to the existing import from `./status-map.js`: + +```typescript +import { createLinearStatusMap, StatusMap } from './status-map.js'; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd packages/outpost/shared && pnpm test -- init.test.ts` +Expected: PASS, all 3 tests green. + +- [ ] **Step 5: Commit** + +```bash +git add packages/outpost/shared/src/sync/init.ts packages/outpost/shared/src/sync/__tests__/init.test.ts +git commit -m "feat(sync): support statusMapOverride in initializeSyncEngine" +``` + +--- + +## Task 4: Wire the worker to register the Linear adapter + +**Files:** +- Create: `apps/worker/src/build-sync-engine.ts` +- Create: `apps/worker/src/__tests__/build-sync-engine.test.ts` +- Modify: `apps/worker/src/index.ts:37-41` + +**Interfaces:** +- Consumes: `loadStatusMap`, `initializeSyncEngine`, `SyncEngine` from `@copilotkit/outpost/shared` (Tasks 2 & 3); `prisma` from `@copilotkit/outpost/db`; `createJob` from `@copilotkit/outpost/queue`. +- Produces: `export async function buildSyncEngine(): Promise` — `index.ts` calls this in place of the bare `new SyncEngine(...)`. + +- [ ] **Step 1: Write the failing test** + +Create `apps/worker/src/__tests__/build-sync-engine.test.ts`: + +```typescript +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockLoadStatusMap = vi.fn(); +const mockInitializeSyncEngine = vi.fn(); + +vi.mock('@copilotkit/outpost/shared', () => ({ + loadStatusMap: (...args: unknown[]) => mockLoadStatusMap(...args), + initializeSyncEngine: (...args: unknown[]) => mockInitializeSyncEngine(...args), +})); + +vi.mock('@copilotkit/outpost/db', () => ({ + prisma: { systemConfig: {}, externalIdentity: {} }, +})); + +vi.mock('@copilotkit/outpost/queue', () => ({ + createJob: vi.fn(), +})); + +import { buildSyncEngine } from '../build-sync-engine.js'; +import { prisma } from '@copilotkit/outpost/db'; +import { createJob } from '@copilotkit/outpost/queue'; + +describe('buildSyncEngine', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('loads the Linear status map and passes it into initializeSyncEngine', async () => { + const fakeStatusMap = { toOutpost: vi.fn() }; + const fakeEngine = { getPlugin: vi.fn() }; + mockLoadStatusMap.mockResolvedValue(fakeStatusMap); + mockInitializeSyncEngine.mockReturnValue(fakeEngine); + + const result = await buildSyncEngine(); + + expect(mockLoadStatusMap).toHaveBeenCalledWith('linear', prisma); + expect(mockInitializeSyncEngine).toHaveBeenCalledWith({ + deps: { prisma, createJob }, + identityDeps: prisma, + statusMapOverride: fakeStatusMap, + }); + expect(result).toBe(fakeEngine); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd apps/worker && pnpm test -- build-sync-engine.test.ts` +Expected: FAIL — `Cannot find module '../build-sync-engine.js'`. + +- [ ] **Step 3: Implement `buildSyncEngine`** + +Create `apps/worker/src/build-sync-engine.ts`: + +```typescript +/** + * Builds the SyncEngine for the TRACKER_SYNC handler, with the Linear + * adapter registered using the persisted status-map config (falling back + * to the hardcoded default when nothing is persisted). GitHub adapter + * registration is not wired here — it needs an authenticated Octokit + * instance that currently only exists inside apps/github-app. + */ + +import { prisma } from '@copilotkit/outpost/db'; +import { createJob } from '@copilotkit/outpost/queue'; +import { loadStatusMap, initializeSyncEngine, type SyncEngine } from '@copilotkit/outpost/shared'; + +export async function buildSyncEngine(): Promise { + const statusMap = await loadStatusMap('linear', prisma as never); + + return initializeSyncEngine({ + deps: { prisma: prisma as never, createJob: createJob as never }, + identityDeps: prisma as never, + statusMapOverride: statusMap, + }); +} +``` + +Modify `apps/worker/src/index.ts` — replace lines 37-41: + +```typescript +// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── + +const syncEngine = new SyncEngine({ prisma: prisma as any, createJob: createJob as any }); +``` + +with: + +```typescript +// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── + +const syncEngine = await buildSyncEngine(); +``` + +Add the import (top of file, near the other `@copilotkit/outpost/shared` import) and remove the now-unused `SyncEngine` import from `@copilotkit/outpost/shared` since it's no longer constructed directly in this file: + +```typescript +import { buildSyncEngine } from './build-sync-engine.js'; +``` + +(Delete `import { SyncEngine } from '@copilotkit/outpost/shared';` — `buildSyncEngine` owns that dependency now.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd apps/worker && pnpm test -- build-sync-engine.test.ts` +Expected: PASS. + +Also run the full worker build to confirm the top-level `await` and import changes compile cleanly: + +Run: `cd apps/worker && pnpm build` +Expected: exits 0. + +- [ ] **Step 5: Commit** + +```bash +git add apps/worker/src/build-sync-engine.ts apps/worker/src/__tests__/build-sync-engine.test.ts apps/worker/src/index.ts +git commit -m "fix(worker): register Linear sync adapter (was never wired up)" +``` + +--- + +## Task 5: Bulk force-sync + +**Files:** +- Modify: `apps/web/src/app/api/sync/force/route.ts` +- Modify: `apps/web/src/__tests__/sync-api.test.ts:274-306` (the `describe('POST /api/sync/force', ...)` block) + +**Interfaces:** +- Consumes: `createJob(JobType.TRACKER_SYNC, payload)` from `@copilotkit/outpost/queue` (already mocked in this test file as `mockCreateJob`); `prisma.ticketExternalLink.findMany` (new mock needed). +- Produces: nothing consumed by later tasks — this is the last task. + +- [ ] **Step 1: Write the failing tests** + +Replace the `describe('POST /api/sync/force', ...)` block (lines 274-306 of `apps/web/src/__tests__/sync-api.test.ts`) with: + +```typescript +describe('POST /api/sync/force', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'ADMIN')); + }); + + it('enqueues status_change and priority_change jobs for every ticket linked to the plugin', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { ticketId: 't-2', plugin: 'linear', ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' } }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 2, jobs: 4 }); + expect(mockCreateJob).toHaveBeenCalledTimes(4); + expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'status_change', + changeData: { status: 'OPEN' }, + }); + expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'priority_change', + changeData: { priority: 'HIGH' }, + }); + }); + + it('returns zero counts and does not call createJob when no tickets are linked', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 0, jobs: 0 }); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('syncs only the given ticket when ticketId is provided', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear', ticketId: 't-1' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 1, jobs: 2 }); + expect(mockTicketExternalLinkFindMany).toHaveBeenCalledWith({ + where: { plugin: 'linear', ticketId: 't-1' }, + include: { ticket: true }, + }); + }); + + it('returns 404 for unknown plugin', async () => { + mockSyncEventFindFirst.mockResolvedValue(null); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'unknown' }); + const res = await forceSync(req as never); + + expect(res.status).toBe(404); + expect(mockTicketExternalLinkFindMany).not.toHaveBeenCalled(); + }); + + it('rejects when plugin is missing', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/force', {}); + const res = await forceSync(req as never); + + expect(res.status).toBe(400); + }); +}); +``` + +Add the new mock near the other `mock*` declarations at the top of the file: + +```typescript +const mockTicketExternalLinkFindMany = vi.fn(); +``` + +Add `ticketExternalLink` alongside `syncEvent`/`externalIdentity`/`systemConfig` inside the `prisma` mock object: + +```typescript + ticketExternalLink: { + findMany: (...args: unknown[]) => mockTicketExternalLinkFindMany(...args), + }, +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd apps/web && pnpm test -- sync-api.test.ts` +Expected: FAIL — all `POST /api/sync/force` bulk/ticketId tests fail with 501 (route still returns "not yet implemented"). + +- [ ] **Step 3: Implement bulk force-sync** + +Replace the entire contents of `apps/web/src/app/api/sync/force/route.ts` with: + +```typescript +import { NextRequest, NextResponse } from 'next/server'; +import { prisma } from '@copilotkit/outpost/db'; +import { createJob, JobType } from '@copilotkit/outpost/queue'; +import { requireAdmin } from '@/lib/require-admin'; + +/** + * POST /api/sync/force + * + * Trigger a force sync for a specific system plugin. Enqueues a + * TRACKER_SYNC job per changed field (status, priority) for every ticket + * currently linked to that plugin, or just one ticket when `ticketId` + * is given. Ticket has no tags/labels field, so label_change is not + * part of a resync. + * + * Body: { plugin: string, ticketId?: string } + */ +export async function POST(request: NextRequest) { + const { error } = await requireAdmin(); + if (error) return error; + + try { + const body = await request.json(); + const plugin = body.plugin; + const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; + + if (!plugin || typeof plugin !== 'string') { + return NextResponse.json( + { error: 'plugin is required' }, + { status: 400 }, + ); + } + + const knownPlugin = await prisma.syncEvent.findFirst({ + where: { + OR: [ + { sourcePlugin: plugin }, + { targetPlugin: plugin }, + ], + }, + }); + + if (!knownPlugin) { + return NextResponse.json( + { error: `Unknown plugin: ${plugin}` }, + { status: 404 }, + ); + } + + const links = await prisma.ticketExternalLink.findMany({ + where: ticketId ? { plugin, ticketId } : { plugin }, + include: { ticket: true }, + }); + + let jobs = 0; + for (const link of links) { + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status: link.ticket.status }, + }); + jobs += 1; + + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority: link.ticket.priority }, + }); + jobs += 1; + } + + return NextResponse.json({ queued: links.length, jobs }); + } catch { + return NextResponse.json( + { error: 'Invalid request body' }, + { status: 400 }, + ); + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd apps/web && pnpm test -- sync-api.test.ts` +Expected: PASS — all tests in the file green, including the pre-existing `GET/PUT /api/sync/mappings` ones from Task 1. + +- [ ] **Step 5: Commit** + +```bash +git add apps/web/src/app/api/sync/force/route.ts apps/web/src/__tests__/sync-api.test.ts +git commit -m "feat(sync): implement bulk force-sync for all tickets linked to a plugin" +``` + +--- + +## Final Verification + +- [ ] **Run the full test suite for every touched package** + +```bash +cd apps/web && pnpm test +cd ../../packages/outpost/shared && pnpm test +cd ../../../apps/worker && pnpm test +``` + +Expected: all green, no regressions in any of the three packages. + +- [ ] **Build check** + +```bash +cd apps/web && pnpm build +cd ../../apps/worker && pnpm build +cd ../../packages/outpost/shared && pnpm build +``` + +Expected: all exit 0. From 51ae71fc3765633590a5ac7bbacd75ae64f3db93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:26:07 -0700 Subject: [PATCH 03/83] feat(sync): persist mapping config to SystemConfig --- apps/web/src/__tests__/api-route-auth.test.ts | 4 ++ apps/web/src/__tests__/sync-api.test.ts | 64 ++++++++++++++++--- apps/web/src/app/api/sync/mappings/route.ts | 61 ++++++++++++------ 3 files changed, 101 insertions(+), 28 deletions(-) diff --git a/apps/web/src/__tests__/api-route-auth.test.ts b/apps/web/src/__tests__/api-route-auth.test.ts index 3e1c6534..476643fb 100644 --- a/apps/web/src/__tests__/api-route-auth.test.ts +++ b/apps/web/src/__tests__/api-route-auth.test.ts @@ -13,6 +13,7 @@ const mockAccountCreate = vi.fn(); const mockAccountFindMany = vi.fn(); const mockTicketGroupBy = vi.fn(); const mockExternalIdentityFindMany = vi.fn(); +const mockSystemConfigFindUnique = vi.fn().mockResolvedValue(null); vi.mock('@copilotkit/outpost/db', () => ({ prisma: { @@ -26,6 +27,9 @@ vi.mock('@copilotkit/outpost/db', () => ({ externalIdentity: { findMany: (...args: unknown[]) => mockExternalIdentityFindMany(...args), }, + systemConfig: { + findUnique: (...args: unknown[]) => mockSystemConfigFindUnique(...args), + }, }, })); diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index cb52b41d..96a5b2f2 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -8,6 +8,8 @@ const mockSyncEventFindUnique = vi.fn(); const mockSyncEventCount = vi.fn(); const mockSyncEventUpdate = vi.fn(); const mockExternalIdentityFindMany = vi.fn(); +const mockSystemConfigFindUnique = vi.fn(); +const mockSystemConfigUpsert = vi.fn(); vi.mock('@copilotkit/outpost/db', () => ({ prisma: { @@ -21,6 +23,10 @@ vi.mock('@copilotkit/outpost/db', () => ({ externalIdentity: { findMany: (...args: unknown[]) => mockExternalIdentityFindMany(...args), }, + systemConfig: { + findUnique: (...args: unknown[]) => mockSystemConfigFindUnique(...args), + upsert: (...args: unknown[]) => mockSystemConfigUpsert(...args), + }, }, })); @@ -232,35 +238,59 @@ describe('GET /api/sync/mappings', () => { mockGetServerSession.mockResolvedValue(userSession('tm-1')); }); - it('returns mapping config', async () => { + it('falls back to defaults when no SystemConfig row exists', async () => { mockExternalIdentityFindMany.mockResolvedValue([]); + mockSystemConfigFindUnique.mockResolvedValue(null); const res = await getMappings(); const body = await res.json(); - expect(body.statusMappings).toBeDefined(); + expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); expect(body.priorityMappings).toBeDefined(); expect(body.identityMappings).toBeDefined(); expect(body.labelRules).toBeDefined(); }); + + it('returns the persisted config when a SystemConfig row exists', async () => { + mockExternalIdentityFindMany.mockResolvedValue([]); + const saved = { + statusMappings: { linear: [{ externalStatus: 'Custom', outpostStatus: 'OPEN' }] }, + priorityMappings: { linear: [] }, + labelRules: { linear: [] }, + }; + mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(saved) }); + + const res = await getMappings(); + const body = await res.json(); + + expect(body.statusMappings).toEqual(saved.statusMappings); + }); }); describe('PUT /api/sync/mappings', () => { beforeEach(() => { vi.clearAllMocks(); - mockGetServerSession.mockResolvedValue(userSession('tm-1')); + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'ADMIN')); }); - it('returns 501 for valid mapping update (persistence not yet implemented)', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, + it('persists a valid mapping update', async () => { + const config = { + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, priorityMappings: { linear: [] }, - }, 'PUT'); - const res = await putMappings(req as never); + }; + mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); - expect(res.status).toBe(501); + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); + const res = await putMappings(req as never); const body = await res.json(); - expect(body.error).toContain('not yet implemented'); + + expect(res.status).toBe(200); + expect(mockSystemConfigUpsert).toHaveBeenCalledWith({ + where: { key: 'sync.mappingConfig' }, + update: { value: JSON.stringify(config) }, + create: { key: 'sync.mappingConfig', value: JSON.stringify(config) }, + }); + expect(body.statusMappings).toEqual(config.statusMappings); }); it('rejects when required fields missing', async () => { @@ -268,6 +298,20 @@ describe('PUT /api/sync/mappings', () => { const res = await putMappings(req as never); expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + + it('requires admin role', async () => { + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); + + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(403); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); }); diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 9fdc07c1..f3dbfb70 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -51,18 +51,36 @@ const DEFAULT_LABEL_RULES: Record { + const row = await prisma.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return null; + try { + return JSON.parse(row.value) as PersistedMappingConfig; + } catch { + return null; + } +} + /** * GET /api/sync/mappings * - * Returns the current mapping configuration. Identity mappings - * are fetched from the ExternalIdentity table; status/priority/label - * mappings are code defaults. + * Returns the current mapping configuration. Identity mappings are always + * fetched live from the ExternalIdentity table. Status/priority/label + * mappings come from the persisted SystemConfig row if one exists, else + * the code defaults. */ export async function GET() { const { error } = await requireSession(); if (error) return error; - // Fetch identity mappings from ExternalIdentity records const identities = await prisma.externalIdentity.findMany({ include: { member: { select: { id: true, name: true } } }, }); @@ -76,22 +94,21 @@ export async function GET() { memberName: ei.member?.name ?? null, })); + const persisted = await readPersistedConfig(); + return NextResponse.json({ - statusMappings: DEFAULT_STATUS_MAPPINGS, - priorityMappings: DEFAULT_PRIORITY_MAPPINGS, + statusMappings: persisted?.statusMappings ?? DEFAULT_STATUS_MAPPINGS, + priorityMappings: persisted?.priorityMappings ?? DEFAULT_PRIORITY_MAPPINGS, identityMappings, - labelRules: DEFAULT_LABEL_RULES, + labelRules: persisted?.labelRules ?? DEFAULT_LABEL_RULES, }); } /** * PUT /api/sync/mappings * - * Update the mapping configuration. - * Body: MappingConfig (statusMappings, priorityMappings required) - * - * Note: This currently validates but does not persist changes to a DB table. - * A settings/config model would be needed for full persistence. + * Persists the mapping configuration as a single JSON row in SystemConfig. + * Body: MappingConfig (statusMappings, priorityMappings required; labelRules optional) */ export async function PUT(request: NextRequest) { const { error } = await requireAdmin(); @@ -100,7 +117,6 @@ export async function PUT(request: NextRequest) { try { const body = await request.json(); - // Basic validation if (!body.statusMappings || !body.priorityMappings) { return NextResponse.json( { error: 'statusMappings and priorityMappings are required' }, @@ -108,11 +124,20 @@ export async function PUT(request: NextRequest) { ); } - // TODO: Persist to a settings/config table once schema supports it - return NextResponse.json( - { error: 'Sync mapping persistence not yet implemented' }, - { status: 501 }, - ); + const config: PersistedMappingConfig = { + statusMappings: body.statusMappings, + priorityMappings: body.priorityMappings, + ...(body.labelRules ? { labelRules: body.labelRules } : {}), + }; + const value = JSON.stringify(config); + + await prisma.systemConfig.upsert({ + where: { key: MAPPING_CONFIG_KEY }, + update: { value }, + create: { key: MAPPING_CONFIG_KEY, value }, + }); + + return NextResponse.json(config); } catch { return NextResponse.json( { error: 'Invalid request body' }, From 9d2ffd268edf42bb62751a6c8fffa2c7c818f2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:26:16 -0700 Subject: [PATCH 04/83] feat(sync): load status map from persisted mapping config --- .../src/sync/__tests__/status-map.test.ts | 55 +++++++++++++++++++ packages/outpost/shared/src/sync/index.ts | 4 +- .../outpost/shared/src/sync/status-map.ts | 52 ++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 packages/outpost/shared/src/sync/__tests__/status-map.test.ts diff --git a/packages/outpost/shared/src/sync/__tests__/status-map.test.ts b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts new file mode 100644 index 00000000..bfbc8d7b --- /dev/null +++ b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect, vi } from 'vitest'; +import { loadStatusMap } from '../status-map.js'; +import { TicketStatus } from '../../types.js'; + +function makeDb(row: { key: string; value: string } | null) { + return { systemConfig: { findUnique: vi.fn().mockResolvedValue(row) } }; +} + +describe('loadStatusMap', () => { + it('falls back to the hardcoded Linear map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + }); + + it('falls back to the hardcoded GitHub map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadStatusMap('github', db); + + expect(map.toOutpost('closed')).toBe(TicketStatus.CLOSED); + }); + + it('builds from persisted config when present for the requested plugin', async () => { + const config = { + statusMappings: { + linear: [{ externalStatus: 'Shipped', outpostStatus: 'RESOLVED' }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Shipped')).toBe(TicketStatus.RESOLVED); + // 'Done' is no longer in the map since the persisted config replaced it entirely + expect(map.toOutpost('Done')).toBe(TicketStatus.OPEN); // StatusMap.toOutpost default fallback + }); + + it('falls back to defaults when persisted config has no entry for this plugin', async () => { + const config = { statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] } }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('github', db); + + expect(map.toOutpost('closed')).toBe(TicketStatus.CLOSED); + }); + + it('falls back to defaults when the persisted value is malformed JSON', async () => { + const db = makeDb({ key: 'sync.mappingConfig', value: 'not json' }); + + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + }); +}); diff --git a/packages/outpost/shared/src/sync/index.ts b/packages/outpost/shared/src/sync/index.ts index 5dfea941..cd8feccd 100644 --- a/packages/outpost/shared/src/sync/index.ts +++ b/packages/outpost/shared/src/sync/index.ts @@ -1,8 +1,8 @@ export * from './types.js'; export { SyncEngine } from './engine.js'; export type { SyncEngineDeps } from './engine.js'; -export { StatusMap, createGitHubStatusMap, createLinearStatusMap } from './status-map.js'; -export type { StatusMappingConfig } from './status-map.js'; +export { StatusMap, createGitHubStatusMap, createLinearStatusMap, loadStatusMap } from './status-map.js'; +export type { StatusMappingConfig, StatusMapDb } from './status-map.js'; export { PriorityMap, createLinearPriorityMap, createGitHubPriorityMap } from './priority-map.js'; export type { PriorityMappingConfig } from './priority-map.js'; export { IdentityMapper } from './identity-map.js'; diff --git a/packages/outpost/shared/src/sync/status-map.ts b/packages/outpost/shared/src/sync/status-map.ts index 91bffe2a..d28db04c 100644 --- a/packages/outpost/shared/src/sync/status-map.ts +++ b/packages/outpost/shared/src/sync/status-map.ts @@ -84,3 +84,55 @@ export function createLinearStatusMap(): StatusMap { Canceled: TicketStatus.CLOSED, }); } + +// ─── Persisted Config Loading ───────────────────────────────────────────── + +/** Must match the key used by apps/web/src/app/api/sync/mappings/route.ts. */ +const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; + +/** Minimal Prisma subset needed to load a persisted mapping config. */ +export interface StatusMapDb { + systemConfig: { + findUnique(args: { where: { key: string } }): Promise<{ key: string; value: string } | null>; + }; +} + +interface PersistedStatusMappingEntry { + externalStatus: string; + outpostStatus: TicketStatus; +} + +/** + * Build a StatusMap for `plugin`, preferring the persisted SystemConfig + * row (written by the /api/sync/mappings dashboard) over the hardcoded + * factory defaults. Falls back to the hardcoded default whenever the + * config row is missing, malformed, or has no entry for this plugin. + */ +export async function loadStatusMap( + plugin: 'linear' | 'github', + db: StatusMapDb, +): Promise { + const fallback = plugin === 'linear' ? createLinearStatusMap() : createGitHubStatusMap(); + + const row = await db.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return fallback; + + let parsed: unknown; + try { + parsed = JSON.parse(row.value); + } catch { + return fallback; + } + + const entries = (parsed as { statusMappings?: Record }) + ?.statusMappings?.[plugin]; + if (!Array.isArray(entries) || entries.length === 0) return fallback; + + const config: StatusMappingConfig = {}; + for (const entry of entries) { + if (entry?.externalStatus && entry?.outpostStatus) { + config[entry.externalStatus] = entry.outpostStatus; + } + } + return Object.keys(config).length > 0 ? new StatusMap(config) : fallback; +} From 9e9094605b0f6a1c88b6180b1c19ceeb1edd43ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:53:02 -0700 Subject: [PATCH 05/83] feat(sync): support statusMapOverride in initializeSyncEngine --- .../shared/src/sync/__tests__/init.test.ts | 69 +++++++++++++++++++ packages/outpost/shared/src/sync/init.ts | 6 +- 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 packages/outpost/shared/src/sync/__tests__/init.test.ts diff --git a/packages/outpost/shared/src/sync/__tests__/init.test.ts b/packages/outpost/shared/src/sync/__tests__/init.test.ts new file mode 100644 index 00000000..eae591f3 --- /dev/null +++ b/packages/outpost/shared/src/sync/__tests__/init.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect, vi } from 'vitest'; +import { initializeSyncEngine } from '../init.js'; +import { StatusMap } from '../status-map.js'; +import { TicketStatus } from '../../types.js'; + +function makeIdentityDeps() { + return { + externalIdentity: { + findUnique: vi.fn(), + findFirst: vi.fn(), + findMany: vi.fn(), + create: vi.fn(), + }, + }; +} + +// SyncEngine requires either an injected echoGuard or prisma.syncEvent to +// construct its internal EchoGuard — stub syncEvent so the engine can be +// constructed in these tests. +function makeSyncEngineDeps() { + return { + prisma: { + syncEvent: { + findFirst: vi.fn().mockResolvedValue(null), + create: vi.fn().mockResolvedValue({ id: 'evt-1' }), + }, + } as never, + createJob: vi.fn(), + }; +} + +describe('initializeSyncEngine', () => { + it('registers no adapters when Linear env vars are missing', () => { + const engine = initializeSyncEngine({ + deps: makeSyncEngineDeps(), + identityDeps: makeIdentityDeps(), + env: {}, + }); + + expect(engine.getPlugin('linear')).toBeUndefined(); + }); + + it('registers the Linear adapter with the default status map when no override is given', () => { + const engine = initializeSyncEngine({ + deps: makeSyncEngineDeps(), + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + }); + + expect(engine.getPlugin('linear')).toBeDefined(); + }); + + it('uses the provided statusMapOverride instead of the hardcoded default', () => { + const customMap = new StatusMap({ Custom: TicketStatus.WAITING_ON_TEAM }); + + const engine = initializeSyncEngine({ + deps: makeSyncEngineDeps(), + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + statusMapOverride: customMap, + }); + + const plugin = engine.getPlugin('linear'); + expect(plugin).toBeDefined(); + // mapStatusToOutpost is the InternalTracker interface method the adapter + // delegates to its injected StatusMap + expect(plugin!.mapStatusToOutpost('Custom')).toBe(TicketStatus.WAITING_ON_TEAM); + }); +}); diff --git a/packages/outpost/shared/src/sync/init.ts b/packages/outpost/shared/src/sync/init.ts index 75e4c38e..2537efa3 100644 --- a/packages/outpost/shared/src/sync/init.ts +++ b/packages/outpost/shared/src/sync/init.ts @@ -9,7 +9,7 @@ import { SyncEngine } from './engine.js'; import type { SyncEngineDeps } from './engine.js'; import { LinearAdapter } from './adapters/linear.js'; -import { createLinearStatusMap } from './status-map.js'; +import { createLinearStatusMap, StatusMap } from './status-map.js'; import { createLinearPriorityMap } from './priority-map.js'; import { createLinearLabelMapper } from './label-map.js'; import { IdentityMapper } from './identity-map.js'; @@ -24,6 +24,8 @@ interface InitOptions { identityDeps?: IdentityMapperDeps; /** Override for environment variables (testing). */ env?: Record; + /** Pre-built StatusMap to use instead of createLinearStatusMap(). */ + statusMapOverride?: StatusMap; } // ─── Initialization ───────────────────────────────────────────────────── @@ -54,7 +56,7 @@ export function initializeSyncEngine(options: InitOptions): SyncEngine { const adapter = new LinearAdapter({ apiKey: linearApiKey, teamId: linearTeamId, - statusMap: createLinearStatusMap(), + statusMap: options.statusMapOverride ?? createLinearStatusMap(), priorityMap: createLinearPriorityMap(), labelMapper: createLinearLabelMapper(), identityMapper, From 5934531b202d53af7c01ed8e843536b21fc254f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:59:11 -0700 Subject: [PATCH 06/83] feat(sync): implement bulk force-sync for all tickets linked to a plugin --- apps/web/src/__tests__/sync-api.test.ts | 61 ++++++++++++++++++++++-- apps/web/src/app/api/sync/force/route.ts | 44 +++++++++++++---- 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 96a5b2f2..12cd10d2 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -10,6 +10,7 @@ const mockSyncEventUpdate = vi.fn(); const mockExternalIdentityFindMany = vi.fn(); const mockSystemConfigFindUnique = vi.fn(); const mockSystemConfigUpsert = vi.fn(); +const mockTicketExternalLinkFindMany = vi.fn(); vi.mock('@copilotkit/outpost/db', () => ({ prisma: { @@ -27,6 +28,9 @@ vi.mock('@copilotkit/outpost/db', () => ({ findUnique: (...args: unknown[]) => mockSystemConfigFindUnique(...args), upsert: (...args: unknown[]) => mockSystemConfigUpsert(...args), }, + ticketExternalLink: { + findMany: (...args: unknown[]) => mockTicketExternalLinkFindMany(...args), + }, }, })); @@ -321,15 +325,63 @@ describe('POST /api/sync/force', () => { mockGetServerSession.mockResolvedValue(userSession('tm-1', 'ADMIN')); }); - it('returns 501 for force sync (bulk sync not yet implemented)', async () => { + it('enqueues status_change and priority_change jobs for every ticket linked to the plugin', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { ticketId: 't-2', plugin: 'linear', ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' } }, + ]); - const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'github' }); + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 2, jobs: 4 }); + expect(mockCreateJob).toHaveBeenCalledTimes(4); + expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'status_change', + changeData: { status: 'OPEN' }, + }); + expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'priority_change', + changeData: { priority: 'HIGH' }, + }); + }); + + it('returns zero counts and does not call createJob when no tickets are linked', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([]); - expect(res.status).toBe(501); + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); const body = await res.json(); - expect(body.error).toContain('not yet implemented'); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 0, jobs: 0 }); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('syncs only the given ticket when ticketId is provided', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear', ticketId: 't-1' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 1, jobs: 2 }); + expect(mockTicketExternalLinkFindMany).toHaveBeenCalledWith({ + where: { plugin: 'linear', ticketId: 't-1' }, + include: { ticket: true }, + }); }); it('returns 404 for unknown plugin', async () => { @@ -339,6 +391,7 @@ describe('POST /api/sync/force', () => { const res = await forceSync(req as never); expect(res.status).toBe(404); + expect(mockTicketExternalLinkFindMany).not.toHaveBeenCalled(); }); it('rejects when plugin is missing', async () => { diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index aefb9679..cf6a01a8 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -1,12 +1,18 @@ import { NextRequest, NextResponse } from 'next/server'; import { prisma } from '@copilotkit/outpost/db'; +import { createJob, JobType } from '@copilotkit/outpost/queue'; import { requireAdmin } from '@/lib/require-admin'; /** * POST /api/sync/force * - * Trigger a force sync for a specific system plugin. - * Body: { plugin: string } + * Trigger a force sync for a specific system plugin. Enqueues a + * TRACKER_SYNC job per changed field (status, priority) for every ticket + * currently linked to that plugin, or just one ticket when `ticketId` + * is given. Ticket has no tags/labels field, so label_change is not + * part of a resync. + * + * Body: { plugin: string, ticketId?: string } */ export async function POST(request: NextRequest) { const { error } = await requireAdmin(); @@ -15,6 +21,7 @@ export async function POST(request: NextRequest) { try { const body = await request.json(); const plugin = body.plugin; + const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; if (!plugin || typeof plugin !== 'string') { return NextResponse.json( @@ -23,7 +30,6 @@ export async function POST(request: NextRequest) { ); } - // Verify the plugin is known by checking if any sync events exist for it const knownPlugin = await prisma.syncEvent.findFirst({ where: { OR: [ @@ -40,13 +46,31 @@ export async function POST(request: NextRequest) { ); } - // TODO: TRACKER_SYNC expects a real ticketId; bulk/full sync needs a - // dedicated FULL_SYNC job type or iteration over all linked tickets. - // For now, return 501 until the handler supports bulk sync. - return NextResponse.json( - { error: 'Bulk force sync not yet implemented' }, - { status: 501 }, - ); + const links = await prisma.ticketExternalLink.findMany({ + where: ticketId ? { plugin, ticketId } : { plugin }, + include: { ticket: true }, + }); + + let jobs = 0; + for (const link of links) { + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status: link.ticket.status }, + }); + jobs += 1; + + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority: link.ticket.priority }, + }); + jobs += 1; + } + + return NextResponse.json({ queued: links.length, jobs }); } catch { return NextResponse.json( { error: 'Invalid request body' }, From a9d95d2de87b9ec3527e61a59d21b4c5199faebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 05:59:31 -0700 Subject: [PATCH 07/83] fix(worker): register Linear sync adapter (was never wired up) --- .../src/__tests__/build-sync-engine.test.ts | 44 +++++++++++++++++++ apps/worker/src/build-sync-engine.ts | 21 +++++++++ apps/worker/src/index.ts | 4 +- 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 apps/worker/src/__tests__/build-sync-engine.test.ts create mode 100644 apps/worker/src/build-sync-engine.ts diff --git a/apps/worker/src/__tests__/build-sync-engine.test.ts b/apps/worker/src/__tests__/build-sync-engine.test.ts new file mode 100644 index 00000000..6aaf118a --- /dev/null +++ b/apps/worker/src/__tests__/build-sync-engine.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockLoadStatusMap = vi.fn(); +const mockInitializeSyncEngine = vi.fn(); + +vi.mock('@copilotkit/outpost/shared', () => ({ + loadStatusMap: (...args: unknown[]) => mockLoadStatusMap(...args), + initializeSyncEngine: (...args: unknown[]) => mockInitializeSyncEngine(...args), +})); + +vi.mock('@copilotkit/outpost/db', () => ({ + prisma: { systemConfig: {}, externalIdentity: {} }, +})); + +vi.mock('@copilotkit/outpost/queue', () => ({ + createJob: vi.fn(), +})); + +import { buildSyncEngine } from '../build-sync-engine.js'; +import { prisma } from '@copilotkit/outpost/db'; +import { createJob } from '@copilotkit/outpost/queue'; + +describe('buildSyncEngine', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('loads the Linear status map and passes it into initializeSyncEngine', async () => { + const fakeStatusMap = { toOutpost: vi.fn() }; + const fakeEngine = { getPlugin: vi.fn() }; + mockLoadStatusMap.mockResolvedValue(fakeStatusMap); + mockInitializeSyncEngine.mockReturnValue(fakeEngine); + + const result = await buildSyncEngine(); + + expect(mockLoadStatusMap).toHaveBeenCalledWith('linear', prisma); + expect(mockInitializeSyncEngine).toHaveBeenCalledWith({ + deps: { prisma, createJob }, + identityDeps: prisma, + statusMapOverride: fakeStatusMap, + }); + expect(result).toBe(fakeEngine); + }); +}); diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts new file mode 100644 index 00000000..08b583dd --- /dev/null +++ b/apps/worker/src/build-sync-engine.ts @@ -0,0 +1,21 @@ +/** + * Builds the SyncEngine for the TRACKER_SYNC handler, with the Linear + * adapter registered using the persisted status-map config (falling back + * to the hardcoded default when nothing is persisted). GitHub adapter + * registration is not wired here — it needs an authenticated Octokit + * instance that currently only exists inside apps/github-app. + */ + +import { prisma } from '@copilotkit/outpost/db'; +import { createJob } from '@copilotkit/outpost/queue'; +import { loadStatusMap, initializeSyncEngine, type SyncEngine } from '@copilotkit/outpost/shared'; + +export async function buildSyncEngine(): Promise { + const statusMap = await loadStatusMap('linear', prisma as never); + + return initializeSyncEngine({ + deps: { prisma: prisma as never, createJob: createJob as never }, + identityDeps: prisma as never, + statusMapOverride: statusMap, + }); +} diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index e80664e8..a870ea02 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -32,11 +32,11 @@ import { handleJobCleanup, createJob, } from '@copilotkit/outpost/queue'; -import { SyncEngine } from '@copilotkit/outpost/shared'; +import { buildSyncEngine } from './build-sync-engine.js'; // ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── -const syncEngine = new SyncEngine({ prisma: prisma as any, createJob: createJob as any }); +const syncEngine = await buildSyncEngine(); const handleTrackerSync = createTrackerSyncHandler(syncEngine); From 1c5882e5202b4134c1cfe5a8cfea2243301e0eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:05:00 -0700 Subject: [PATCH 08/83] fix(worker): remove unused createJob import --- apps/worker/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index a870ea02..d7458596 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -30,7 +30,6 @@ import { handleHubSpotSync, createTrackerSyncHandler, handleJobCleanup, - createJob, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; From 491fb732215f798e816b9aa17e8d86d762fab824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 06:12:33 -0700 Subject: [PATCH 09/83] docs: fix spec to match no-tags schema (2 jobs, not 3) --- ...apping-persistence-and-bulk-force-sync-design.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md index 71d6c9be..4fb5ecb6 100644 --- a/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md +++ b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md @@ -76,11 +76,12 @@ caller, not looked up from a plugin-level map, per `tracker-sync.ts:181-188`). same "current values" push described below, since it's currently also unimplemented). - `ticketId` absent → bulk mode: 1. `prisma.ticketExternalLink.findMany({ where: { plugin }, include: { ticket: true } })` - 2. For each linked ticket, enqueue three `TRACKER_SYNC` jobs (reusing the existing job + 2. For each linked ticket, enqueue two `TRACKER_SYNC` jobs (reusing the existing job type/handler, untouched) via `createJob`: - `action: 'status_change'`, `changeData: { status: ticket.status }` - `action: 'priority_change'`, `changeData: { priority: ticket.priority }` - - `action: 'label_change'`, `changeData: { labels: ticket.tags }` (skip if empty) + (`Ticket` has no tags/labels field in the current schema, so `label_change` is not + part of bulk resync — there's no source value to push.) 3. Return `{ queued: , jobs: }`. The route only inserts jobs (cheap Postgres writes); the worker performs the actual pushes @@ -99,10 +100,10 @@ Per repo convention (Vitest, red-green, webhook/job tests use mocked Prisma): missing `statusMappings`/`priorityMappings`. - `status-map.ts` `loadStatusMap`: test it builds from a mocked `SystemConfig` row; test it falls back to the hardcoded factory when the row or plugin key is absent. -- `force/route.ts`: test bulk mode enqueues 3 jobs per linked ticket (mock - `ticketExternalLink.findMany` returning N tickets, assert `createJob` called 3N times); - test single-ticket mode still works; test unknown plugin still 404s; test empty `tags` - skips the label job. +- `force/route.ts`: test bulk mode enqueues 2 jobs per linked ticket (mock + `ticketExternalLink.findMany` returning N tickets, assert `createJob` called 2N times); + test single-ticket mode still works; test unknown plugin still 404s; test zero linked + tickets returns `{ queued: 0, jobs: 0 }` without calling `createJob`. ## Files touched From d3742b6bb37cdc1fbb2757f8ddfe944c62f18668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:37:46 -0700 Subject: [PATCH 10/83] fix(sync): validate persisted outpostStatus against TicketStatus enum loadStatusMap only truthiness-checked entry.outpostStatus before casting it straight into the StatusMap config. A persisted row with a typo'd or corrupted outpostStatus (e.g. 'GARBAGE') passed through untouched, so toOutpost() later returned that literal garbage string instead of the safe TicketStatus.OPEN fallback -- silently corrupting downstream ticket status logic and, via the sync push path, risking a bad status write to Linear. Fix: reject any entry whose outpostStatus is not a real TicketStatus enum member (Object.values(TicketStatus).includes(...)) before adding it to the config. If every entry for a plugin is invalid, the existing "entries present but map empty -> fallback" logic already covers it. Call-site enumeration (grep -rn "loadStatusMap" packages/outpost apps): - packages/outpost/shared/src/sync/index.ts:4 -- re-export only, signature unchanged, unaffected. - packages/outpost/shared/src/sync/__tests__/status-map.test.ts -- updated with 2 new tests (invalid entry skipped / all-invalid fallback), all 7 tests pass. - apps/worker/src/build-sync-engine.ts:14 -- consumes the returned StatusMap object only; return type and success-path behavior unchanged, stricter filtering only improves correctness for it. - apps/worker/src/__tests__/build-sync-engine.test.ts:7 -- mocks loadStatusMap entirely, never exercises real implementation, unaffected. --- .../src/sync/__tests__/status-map.test.ts | 33 +++++++++++++++++++ .../outpost/shared/src/sync/status-map.ts | 6 +++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/outpost/shared/src/sync/__tests__/status-map.test.ts b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts index bfbc8d7b..e4f0d0f6 100644 --- a/packages/outpost/shared/src/sync/__tests__/status-map.test.ts +++ b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts @@ -52,4 +52,37 @@ describe('loadStatusMap', () => { expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); }); + + it('skips a persisted entry whose outpostStatus is not a valid TicketStatus enum value', async () => { + const config = { + statusMappings: { + linear: [ + { externalStatus: 'Shipped', outpostStatus: 'RESOLVED' }, + { externalStatus: 'Custom', outpostStatus: 'NOT_A_REAL_STATUS' }, + ], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('linear', db); + + expect(map.toOutpost('Shipped')).toBe(TicketStatus.RESOLVED); + // Invalid entry must not pass through as a literal garbage string. + expect(map.toOutpost('Custom')).toBe(TicketStatus.OPEN); + }); + + it('falls back to the hardcoded default map when every persisted entry for a plugin is invalid', async () => { + const config = { + statusMappings: { + linear: [{ externalStatus: 'Custom', outpostStatus: 'NOT_A_REAL_STATUS' }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadStatusMap('linear', db); + + // All entries filtered out -> falls back to createLinearStatusMap() defaults. + expect(map.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + expect(map.toOutpost('Custom')).toBe(TicketStatus.OPEN); + }); }); diff --git a/packages/outpost/shared/src/sync/status-map.ts b/packages/outpost/shared/src/sync/status-map.ts index d28db04c..4a0b9e15 100644 --- a/packages/outpost/shared/src/sync/status-map.ts +++ b/packages/outpost/shared/src/sync/status-map.ts @@ -130,7 +130,11 @@ export async function loadStatusMap( const config: StatusMappingConfig = {}; for (const entry of entries) { - if (entry?.externalStatus && entry?.outpostStatus) { + if ( + entry?.externalStatus && + entry?.outpostStatus && + Object.values(TicketStatus).includes(entry.outpostStatus as TicketStatus) + ) { config[entry.externalStatus] = entry.outpostStatus; } } From 6d67034a77b340f05c1bcd204e8520648638962b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:38:01 -0700 Subject: [PATCH 11/83] fix(sync): don't swallow errors as 400, recognize plugin via links Two CR findings on the same function, fixed together (both live in POST /api/sync/force): - Narrowed try/catch to only cover request.json() parsing, so a mid-loop DB/queue failure propagates instead of being mislabeled "Invalid request body" (400). - The "known plugin" gate now also accepts a plugin with a real TicketExternalLink, not just a prior SyncEvent -- previously a plugin's very first force-sync (real links, zero sync history) 404'd. Call-site enumeration: POST is framework-invoked by Next.js; no other code calls it directly. Signature/return type unchanged. --- apps/web/src/__tests__/sync-api.test.ts | 44 +++++++++++ apps/web/src/app/api/sync/force/route.ts | 95 +++++++++++++----------- 2 files changed, 94 insertions(+), 45 deletions(-) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 12cd10d2..e1168154 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -11,6 +11,7 @@ const mockExternalIdentityFindMany = vi.fn(); const mockSystemConfigFindUnique = vi.fn(); const mockSystemConfigUpsert = vi.fn(); const mockTicketExternalLinkFindMany = vi.fn(); +const mockTicketExternalLinkFindFirst = vi.fn(); vi.mock('@copilotkit/outpost/db', () => ({ prisma: { @@ -30,6 +31,7 @@ vi.mock('@copilotkit/outpost/db', () => ({ }, ticketExternalLink: { findMany: (...args: unknown[]) => mockTicketExternalLinkFindMany(...args), + findFirst: (...args: unknown[]) => mockTicketExternalLinkFindFirst(...args), }, }, })); @@ -323,6 +325,7 @@ describe('POST /api/sync/force', () => { beforeEach(() => { vi.clearAllMocks(); mockGetServerSession.mockResolvedValue(userSession('tm-1', 'ADMIN')); + mockTicketExternalLinkFindFirst.mockResolvedValue(null); }); it('enqueues status_change and priority_change jobs for every ticket linked to the plugin', async () => { @@ -386,6 +389,7 @@ describe('POST /api/sync/force', () => { it('returns 404 for unknown plugin', async () => { mockSyncEventFindFirst.mockResolvedValue(null); + mockTicketExternalLinkFindFirst.mockResolvedValue(null); const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'unknown' }); const res = await forceSync(req as never); @@ -394,10 +398,50 @@ describe('POST /api/sync/force', () => { expect(mockTicketExternalLinkFindMany).not.toHaveBeenCalled(); }); + it('recognizes a plugin via TicketExternalLink even with no prior SyncEvent', async () => { + mockSyncEventFindFirst.mockResolvedValue(null); + mockTicketExternalLinkFindFirst.mockResolvedValue({ id: 'link-1', plugin: 'linear' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(body).toEqual({ queued: 1, jobs: 2 }); + }); + it('rejects when plugin is missing', async () => { const req = makeJsonRequest('http://localhost:3000/api/sync/force', {}); const res = await forceSync(req as never); expect(res.status).toBe(400); }); + + it('still returns 400 "Invalid request body" for malformed JSON', async () => { + const req = new NextRequest('http://localhost:3000/api/sync/force', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: 'not json', + }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(400); + expect(body.error).toBe('Invalid request body'); + }); + + it('does not mislabel a mid-loop DB/queue error as "Invalid request body"', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + ]); + mockCreateJob.mockRejectedValueOnce(new Error('db down')); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + + await expect(forceSync(req as never)).rejects.toThrow('db down'); + }); }); diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index cf6a01a8..0026f06e 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -18,63 +18,68 @@ export async function POST(request: NextRequest) { const { error } = await requireAdmin(); if (error) return error; + let body: { plugin?: unknown; ticketId?: unknown }; try { - const body = await request.json(); - const plugin = body.plugin; - const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; + body = await request.json(); + } catch { + return NextResponse.json( + { error: 'Invalid request body' }, + { status: 400 }, + ); + } - if (!plugin || typeof plugin !== 'string') { - return NextResponse.json( - { error: 'plugin is required' }, - { status: 400 }, - ); - } + const plugin = body.plugin; + const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; - const knownPlugin = await prisma.syncEvent.findFirst({ + if (!plugin || typeof plugin !== 'string') { + return NextResponse.json( + { error: 'plugin is required' }, + { status: 400 }, + ); + } + + const [knownPlugin, hasLinks] = await Promise.all([ + prisma.syncEvent.findFirst({ where: { OR: [ { sourcePlugin: plugin }, { targetPlugin: plugin }, ], }, - }); - - if (!knownPlugin) { - return NextResponse.json( - { error: `Unknown plugin: ${plugin}` }, - { status: 404 }, - ); - } + }), + prisma.ticketExternalLink.findFirst({ where: { plugin } }), + ]); - const links = await prisma.ticketExternalLink.findMany({ - where: ticketId ? { plugin, ticketId } : { plugin }, - include: { ticket: true }, - }); + if (!knownPlugin && !hasLinks) { + return NextResponse.json( + { error: `Unknown plugin: ${plugin}` }, + { status: 404 }, + ); + } - let jobs = 0; - for (const link of links) { - await createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'status_change', - changeData: { status: link.ticket.status }, - }); - jobs += 1; + const links = await prisma.ticketExternalLink.findMany({ + where: ticketId ? { plugin, ticketId } : { plugin }, + include: { ticket: true }, + }); - await createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'priority_change', - changeData: { priority: link.ticket.priority }, - }); - jobs += 1; - } + let jobs = 0; + for (const link of links) { + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status: link.ticket.status }, + }); + jobs += 1; - return NextResponse.json({ queued: links.length, jobs }); - } catch { - return NextResponse.json( - { error: 'Invalid request body' }, - { status: 400 }, - ); + await createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority: link.ticket.priority }, + }); + jobs += 1; } + + return NextResponse.json({ queued: links.length, jobs }); } From 143ebcf7030b8858f92ede4000b884b659263d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:49:12 -0700 Subject: [PATCH 12/83] fix(sync): validate mapping config shape before persisting PUT /api/sync/mappings only truthiness-checked statusMappings and priorityMappings, so a caller could persist a garbage-shaped value (wrong type, unknown outpostStatus/outpostPriority) and GET would echo it back as valid config. Adds isValidMappingShape() checking both fields are { [plugin]: Array<{external*, outpost*}> } with outpost* values restricted to the real TicketStatus/TicketPriority enum members, imported from @copilotkit/outpost/shared (not /db, which only exports the prisma client). Call-site enumeration: PUT is framework-invoked; no other code calls it directly. isValidMappingShape is new, no other call sites. --- apps/web/src/__tests__/sync-api.test.ts | 22 +++++++++++ apps/web/src/app/api/sync/mappings/route.ts | 44 +++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index e1168154..8b0bda0c 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -307,6 +307,28 @@ describe('PUT /api/sync/mappings', () => { expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); + it('rejects statusMappings of the wrong type entirely', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: 'garbage', + priorityMappings: { linear: [] }, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + + it('rejects a mapping with an unknown outpostStatus value', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'NOT_REAL' }] }, + priorityMappings: { linear: [] }, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + it('requires admin role', async () => { mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index f3dbfb70..f1db1aa3 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { requireSession, requireAdmin } from '@/lib/require-admin'; import { prisma } from '@copilotkit/outpost/db'; +import { TicketStatus, TicketPriority } from '@copilotkit/outpost/shared'; /** * Default mapping configuration. In a full implementation this would @@ -104,6 +105,35 @@ export async function GET() { }); } +/** + * Validates that `value` matches the expected mapping shape: + * a plain object whose values are arrays of + * `{ externalStatus: string; outpostStatus: }` + * (or the priority equivalent, keyed `externalPriority`/`outpostPriority`). + */ +function isValidMappingShape(value: unknown, validOutpostValues: string[]): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + + return Object.values(value as Record).every((entries) => { + if (!Array.isArray(entries)) return false; + + return entries.every((entry) => { + if (typeof entry !== 'object' || entry === null) return false; + const record = entry as Record; + const externalKey = 'externalStatus' in record ? 'externalStatus' : 'externalPriority'; + const outpostKey = 'externalStatus' in record ? 'outpostStatus' : 'outpostPriority'; + + return ( + typeof record[externalKey] === 'string' && + typeof record[outpostKey] === 'string' && + validOutpostValues.includes(record[outpostKey] as string) + ); + }); + }); +} + /** * PUT /api/sync/mappings * @@ -124,6 +154,20 @@ export async function PUT(request: NextRequest) { ); } + if (!isValidMappingShape(body.statusMappings, Object.values(TicketStatus))) { + return NextResponse.json( + { error: 'statusMappings has invalid shape or unknown outpostStatus value' }, + { status: 400 }, + ); + } + + if (!isValidMappingShape(body.priorityMappings, Object.values(TicketPriority))) { + return NextResponse.json( + { error: 'priorityMappings has invalid shape or unknown outpostPriority value' }, + { status: 400 }, + ); + } + const config: PersistedMappingConfig = { statusMappings: body.statusMappings, priorityMappings: body.priorityMappings, From 17b99a21446bf9200fc38d0417d2ec0423ca959d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:36:36 -0700 Subject: [PATCH 13/83] test(sync): cover requireAdmin gate on POST /api/sync/force --- apps/web/src/__tests__/sync-api.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 8b0bda0c..8f7f6e8a 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -466,4 +466,16 @@ describe('POST /api/sync/force', () => { await expect(forceSync(req as never)).rejects.toThrow('db down'); }); + + it('requires admin role', async () => { + mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); + + expect(res.status).toBe(403); + expect(mockSyncEventFindFirst).not.toHaveBeenCalled(); + expect(mockTicketExternalLinkFindMany).not.toHaveBeenCalled(); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); }); From 0499c1f52e720e5262e2701e1034e00dbb6b9100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:37:46 -0700 Subject: [PATCH 14/83] test(sync): cover malformed-JSON fallback in GET /api/sync/mappings --- apps/web/src/__tests__/sync-api.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 8f7f6e8a..4f1ac23a 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -271,6 +271,19 @@ describe('GET /api/sync/mappings', () => { expect(body.statusMappings).toEqual(saved.statusMappings); }); + + it('falls back to defaults when the persisted SystemConfig value is malformed JSON', async () => { + mockExternalIdentityFindMany.mockResolvedValue([]); + mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: 'not valid json {{{' }); + + const res = await getMappings(); + const body = await res.json(); + + expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); + expect(body.priorityMappings).toBeDefined(); + expect(body.identityMappings).toBeDefined(); + expect(body.labelRules).toBeDefined(); + }); }); describe('PUT /api/sync/mappings', () => { From 104163aed4e7304f791003cd6e08f954fc670bf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:38:34 -0700 Subject: [PATCH 15/83] docs(sync): fix stale/inaccurate docstrings on new sync code - force/route.ts: "per changed field" implied change detection; the handler unconditionally enqueues both jobs. - build-sync-engine.ts: presented Linear registration as unconditional; it's env-gated (LINEAR_API_KEY + LINEAR_TEAM_ID). - mappings/route.ts: top-of-file comment said mappings are "kept as code defaults," directly contradicted by the PUT/GET persistence logic added right below it. Comment-only, no behavior change, no test required. --- apps/web/src/app/api/sync/force/route.ts | 10 +++++----- apps/web/src/app/api/sync/mappings/route.ts | 7 +++---- apps/worker/src/build-sync-engine.ts | 12 +++++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index 0026f06e..d50302e9 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -6,11 +6,11 @@ import { requireAdmin } from '@/lib/require-admin'; /** * POST /api/sync/force * - * Trigger a force sync for a specific system plugin. Enqueues a - * TRACKER_SYNC job per changed field (status, priority) for every ticket - * currently linked to that plugin, or just one ticket when `ticketId` - * is given. Ticket has no tags/labels field, so label_change is not - * part of a resync. + * Trigger a force sync for a specific system plugin. Unconditionally + * enqueues status_change and priority_change TRACKER_SYNC jobs (no + * change detection) for every ticket currently linked to that plugin, + * or just one ticket when `ticketId` is given. Ticket has no + * tags/labels field, so label_change is not part of a resync. * * Body: { plugin: string, ticketId?: string } */ diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index f1db1aa3..838cdbbe 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -4,10 +4,9 @@ import { prisma } from '@copilotkit/outpost/db'; import { TicketStatus, TicketPriority } from '@copilotkit/outpost/shared'; /** - * Default mapping configuration. In a full implementation this would - * come from a dedicated settings/config table. For now we derive - * identity mappings from ExternalIdentity records and keep - * status/priority/label mappings as code defaults. + * Default mapping configuration, used when nothing has been persisted + * to SystemConfig yet (see PUT below, which persists real overrides). + * Identity mappings always come live from ExternalIdentity records. */ const DEFAULT_STATUS_MAPPINGS: Record> = { linear: [ diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index 08b583dd..cf34b287 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -1,9 +1,11 @@ /** - * Builds the SyncEngine for the TRACKER_SYNC handler, with the Linear - * adapter registered using the persisted status-map config (falling back - * to the hardcoded default when nothing is persisted). GitHub adapter - * registration is not wired here — it needs an authenticated Octokit - * instance that currently only exists inside apps/github-app. + * Builds the SyncEngine for the TRACKER_SYNC handler. Registers the + * Linear adapter, using the persisted status-map config (falling back + * to the hardcoded default when nothing is persisted), only when + * LINEAR_API_KEY and LINEAR_TEAM_ID are set — absent those, the engine + * has zero registered adapters and TRACKER_SYNC jobs no-op. GitHub + * adapter registration is not wired here — it needs an authenticated + * Octokit instance that currently only exists inside apps/github-app. */ import { prisma } from '@copilotkit/outpost/db'; From 0b9517893f938b9e50da69d3b5a72a960173b870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:41:06 -0700 Subject: [PATCH 16/83] fix(sync): reject empty mapping objects instead of silently wiping config isValidMappingShape({}, ...) returned true because Object.values({}).every(...) is vacuously true. A PUT with {statusMappings: {}, priorityMappings: {}} passed shape validation, got persisted, and GET then returned {} for both fields instead of falling back to defaults -- silently wiping any real config. Now require at least one own key on the top-level object before checking entries, so a fully-empty mapping object is rejected (400) while a config that legitimately maps some plugins and omits others (e.g. {linear: [...]} with no github key) still passes. Call-site enumeration (grep -n "isValidMappingShape" route.ts): 114: function isValidMappingShape(...) 161: if (!isValidMappingShape(body.statusMappings, Object.values(TicketStatus))) 168: if (!isValidMappingShape(body.priorityMappings, Object.values(TicketPriority))) Exactly two call sites (statusMappings, priorityMappings), both unaffected for non-empty inputs -- only a fully-empty {} now fails at each site. --- apps/web/src/__tests__/sync-api.test.ts | 11 +++++++++++ apps/web/src/app/api/sync/mappings/route.ts | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 4f1ac23a..328cf691 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -354,6 +354,17 @@ describe('PUT /api/sync/mappings', () => { expect(res.status).toBe(403); expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); + + it('rejects completely empty statusMappings/priorityMappings objects instead of silently wiping config', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: {}, + priorityMappings: {}, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); }); describe('POST /api/sync/force', () => { diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 838cdbbe..545229db 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -115,6 +115,10 @@ function isValidMappingShape(value: unknown, validOutpostValues: string[]): bool return false; } + if (Object.keys(value as Record).length === 0) { + return false; + } + return Object.values(value as Record).every((entries) => { if (!Array.isArray(entries)) return false; From fb0986e0f8ae5e344ba8d2b6fcab8848cb3a6133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:43:30 -0700 Subject: [PATCH 17/83] fix(sync): validate labelRules shape before persisting PUT /api/sync/mappings validated statusMappings/priorityMappings shape but persisted body.labelRules verbatim with zero validation, so a caller sending labelRules: "garbage" or labelRules: 42 would have it stored and later echoed back by GET as if it matched Record>. Added isValidLabelRulesShape (mirrors isValidMappingShape's object/array checks but drops the enum constraint since outpostPrefix can be any string, including empty). PUT now returns 400 without calling prisma.systemConfig.upsert when labelRules is present but malformed. Call-site enumeration: grepped labelRules across apps/web/src and packages/outpost. Only other reads are mapping-editor.tsx (reads the already-typed GET response into client state) and mock-sync.ts (unrelated mock/dev data). Nothing else reads body.labelRules directly, so nothing depended on the previous unvalidated pass-through. Tests: added a red/green case (garbage labelRules -> 400, upsert not called; confirmed failing against old code, passing after) and a green-only case establishing missing coverage for the valid labelRules persistence path. --- apps/web/src/__tests__/sync-api.test.ts | 33 ++++++++++++++++++++ apps/web/src/app/api/sync/mappings/route.ts | 34 +++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 328cf691..6de7d1ca 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -365,6 +365,39 @@ describe('PUT /api/sync/mappings', () => { expect(res.status).toBe(400); expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); + + it('rejects labelRules of the wrong type', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + labelRules: 'garbage', + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + + it('persists a valid labelRules update', async () => { + const config = { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + labelRules: { linear: [{ externalPrefix: 'Priority: ', outpostPrefix: '' }] }, + }; + mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); + const res = await putMappings(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + expect(mockSystemConfigUpsert).toHaveBeenCalledWith({ + where: { key: 'sync.mappingConfig' }, + update: { value: JSON.stringify(config) }, + create: { key: 'sync.mappingConfig', value: JSON.stringify(config) }, + }); + expect(body.labelRules).toEqual(config.labelRules); + }); }); describe('POST /api/sync/force', () => { diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 545229db..2520066b 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -137,6 +137,33 @@ function isValidMappingShape(value: unknown, validOutpostValues: string[]): bool }); } +/** + * Validates that `value` matches the expected labelRules shape: + * a plain object whose values are arrays of + * `{ externalPrefix: string; outpostPrefix: string }`. Unlike + * `isValidMappingShape`, there is no enum constraint on `outpostPrefix` — + * any string (including empty string) is valid. + */ +function isValidLabelRulesShape(value: unknown): boolean { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return false; + } + + return Object.values(value as Record).every((entries) => { + if (!Array.isArray(entries)) return false; + + return entries.every((entry) => { + if (typeof entry !== 'object' || entry === null) return false; + const record = entry as Record; + + return ( + typeof record.externalPrefix === 'string' && + typeof record.outpostPrefix === 'string' + ); + }); + }); +} + /** * PUT /api/sync/mappings * @@ -171,6 +198,13 @@ export async function PUT(request: NextRequest) { ); } + if (body.labelRules !== undefined && !isValidLabelRulesShape(body.labelRules)) { + return NextResponse.json( + { error: 'labelRules has invalid shape' }, + { status: 400 }, + ); + } + const config: PersistedMappingConfig = { statusMappings: body.statusMappings, priorityMappings: body.priorityMappings, From 8ebc5989a334a273f809ddcd83506d5aecb5c28d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:45:03 -0700 Subject: [PATCH 18/83] fix(sync): don't swallow DB errors as 400 in mappings PUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PUT handler in apps/web/src/app/api/sync/mappings/route.ts wrapped the entire body — including prisma.systemConfig.upsert — in a single try/catch that mapped any failure to a 400 "Invalid request body". A DB failure (connection issue, constraint violation) was therefore mislabeled as a client error instead of propagating as a 500. Same bug class already fixed in the sibling apps/web/src/app/api/sync/force/route.ts (narrowed try/catch to only wrap request.json() parsing) but not mirrored here. Narrowed the try/catch to cover only `await request.json()`; the shape validation checks and the prisma.systemConfig.upsert call now run outside that catch, so a DB failure propagates as an unhandled error (Next.js turns it into a 500) instead of a misleading 400. Added a covering test mirroring the equivalent force-route test: mocks systemConfig.upsert to reject with an Error and asserts the PUT handler rejects/throws rather than returning a 400. Verified red (failed against the old code, which returned a 400 response) then green (passes with the fix) before committing. Call-site enumeration: PUT is a framework-invoked Next.js route handler. Only call site in the codebase is the frontend fetch('/api/sync/mappings', { method: 'PUT', ... }) in apps/web/src/app/sync/mappings/page.tsx — no direct code call-sites to the exported PUT function besides the test file. Trivially clean. --- apps/web/src/__tests__/sync-api.test.ts | 12 +++ apps/web/src/app/api/sync/mappings/route.ts | 85 +++++++++++---------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 6de7d1ca..79980f28 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -398,6 +398,18 @@ describe('PUT /api/sync/mappings', () => { }); expect(body.labelRules).toEqual(config.labelRules); }); + + it('does not mislabel a DB write failure as "Invalid request body"', async () => { + const config = { + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { linear: [] }, + }; + mockSystemConfigUpsert.mockRejectedValueOnce(new Error('db down')); + + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); + + await expect(putMappings(req as never)).rejects.toThrow('db down'); + }); }); describe('POST /api/sync/force', () => { diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 2520066b..7c7a8bad 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -174,55 +174,56 @@ export async function PUT(request: NextRequest) { const { error } = await requireAdmin(); if (error) return error; + let body: { statusMappings?: unknown; priorityMappings?: unknown; labelRules?: unknown }; try { - const body = await request.json(); - - if (!body.statusMappings || !body.priorityMappings) { - return NextResponse.json( - { error: 'statusMappings and priorityMappings are required' }, - { status: 400 }, - ); - } + body = await request.json(); + } catch { + return NextResponse.json( + { error: 'Invalid request body' }, + { status: 400 }, + ); + } - if (!isValidMappingShape(body.statusMappings, Object.values(TicketStatus))) { - return NextResponse.json( - { error: 'statusMappings has invalid shape or unknown outpostStatus value' }, - { status: 400 }, - ); - } + if (!body.statusMappings || !body.priorityMappings) { + return NextResponse.json( + { error: 'statusMappings and priorityMappings are required' }, + { status: 400 }, + ); + } - if (!isValidMappingShape(body.priorityMappings, Object.values(TicketPriority))) { - return NextResponse.json( - { error: 'priorityMappings has invalid shape or unknown outpostPriority value' }, - { status: 400 }, - ); - } + if (!isValidMappingShape(body.statusMappings, Object.values(TicketStatus))) { + return NextResponse.json( + { error: 'statusMappings has invalid shape or unknown outpostStatus value' }, + { status: 400 }, + ); + } - if (body.labelRules !== undefined && !isValidLabelRulesShape(body.labelRules)) { - return NextResponse.json( - { error: 'labelRules has invalid shape' }, - { status: 400 }, - ); - } - - const config: PersistedMappingConfig = { - statusMappings: body.statusMappings, - priorityMappings: body.priorityMappings, - ...(body.labelRules ? { labelRules: body.labelRules } : {}), - }; - const value = JSON.stringify(config); - - await prisma.systemConfig.upsert({ - where: { key: MAPPING_CONFIG_KEY }, - update: { value }, - create: { key: MAPPING_CONFIG_KEY, value }, - }); + if (!isValidMappingShape(body.priorityMappings, Object.values(TicketPriority))) { + return NextResponse.json( + { error: 'priorityMappings has invalid shape or unknown outpostPriority value' }, + { status: 400 }, + ); + } - return NextResponse.json(config); - } catch { + if (body.labelRules !== undefined && !isValidLabelRulesShape(body.labelRules)) { return NextResponse.json( - { error: 'Invalid request body' }, + { error: 'labelRules has invalid shape' }, { status: 400 }, ); } + + const config: PersistedMappingConfig = { + statusMappings: body.statusMappings as PersistedMappingConfig['statusMappings'], + priorityMappings: body.priorityMappings as PersistedMappingConfig['priorityMappings'], + ...(body.labelRules ? { labelRules: body.labelRules as PersistedMappingConfig['labelRules'] } : {}), + }; + const value = JSON.stringify(config); + + await prisma.systemConfig.upsert({ + where: { key: MAPPING_CONFIG_KEY }, + update: { value }, + create: { key: MAPPING_CONFIG_KEY, value }, + }); + + return NextResponse.json(config); } From 85b45ef5fe6a2bc0d824f30868823b44de9ca5fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:41:48 -0700 Subject: [PATCH 19/83] test(sync): cover invalid outpostPriority rejection in PUT /api/sync/mappings --- apps/web/src/__tests__/sync-api.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 79980f28..521000f0 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -342,6 +342,17 @@ describe('PUT /api/sync/mappings', () => { expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); + it('rejects a mapping with an unknown outpostPriority value', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [] }, + priorityMappings: { linear: [{ externalPriority: 'X', outpostPriority: 'NOT_REAL' }] }, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + it('requires admin role', async () => { mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); From 1dc575dcd72835e7e82e645a292a6bc7acf96118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:16:43 -0700 Subject: [PATCH 20/83] =?UTF-8?q?docs(sync):=20fix=20build-sync-engine=20d?= =?UTF-8?q?ocstring=20=E2=80=94=20jobs=20fail+retry,=20not=20no-op?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-3 CR finding: the docstring claimed TRACKER_SYNC jobs "no-op" when no adapter is registered, but the handler returns {success:false, error:'Plugin "..." is not registered'} — the job fails and retries per the queue's normal policy, it doesn't silently do nothing. Comment-only, no behavior change. --- apps/worker/src/build-sync-engine.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index cf34b287..fca1a700 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -3,7 +3,9 @@ * Linear adapter, using the persisted status-map config (falling back * to the hardcoded default when nothing is persisted), only when * LINEAR_API_KEY and LINEAR_TEAM_ID are set — absent those, the engine - * has zero registered adapters and TRACKER_SYNC jobs no-op. GitHub + * has zero registered adapters and TRACKER_SYNC jobs fail with "Plugin + * ... is not registered" (retried per the queue's normal retry policy, + * not silently dropped). GitHub * adapter registration is not wired here — it needs an authenticated * Octokit instance that currently only exists inside apps/github-app. */ From f74c4fa11def4109cf59b0a8820148366eded788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:18:34 -0700 Subject: [PATCH 21/83] fix(sync): reject empty labelRules object, matching statusMappings/priorityMappings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isValidMappingShape already rejected {} to stop empty config from silently wiping stored statusMappings/priorityMappings. isValidLabelRulesShape never got the same guard, so Object.values({}).every(...) vacuously returned true and PUT with labelRules: {} passed validation and persisted an empty labelRules object. Flagged independently by 3 reviewers across two CR rounds. Call-site enumeration: isValidLabelRulesShape is called exactly once (route.ts:212), guarded by `body.labelRules !== undefined` — undefined (optional labelRules) still bypasses validation entirely, unaffected. A non-empty valid labelRules object still passes since the new check only rejects zero-key objects. --- apps/web/src/__tests__/sync-api.test.ts | 12 ++++++++++++ apps/web/src/app/api/sync/mappings/route.ts | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 521000f0..6289cfb2 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -410,6 +410,18 @@ describe('PUT /api/sync/mappings', () => { expect(body.labelRules).toEqual(config.labelRules); }); + it('rejects a completely empty labelRules object instead of silently wiping config', async () => { + const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + labelRules: {}, + }, 'PUT'); + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + it('does not mislabel a DB write failure as "Invalid request body"', async () => { const config = { statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 7c7a8bad..6674b268 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -149,6 +149,10 @@ function isValidLabelRulesShape(value: unknown): boolean { return false; } + if (Object.keys(value as Record).length === 0) { + return false; + } + return Object.values(value as Record).every((entries) => { if (!Array.isArray(entries)) return false; From b61c02ac5572c602d4c8e69c3fd253a562eac84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 15 Jul 2026 06:24:07 -0700 Subject: [PATCH 22/83] docs: require cr-loop review before every push Standing rule going forward, not case-by-case: run copilotkit-internal:cr-loop on the diff before pushing any non-trivial change. Prompted by this branch's CR loop catching real bugs across 3 rounds that would otherwise have shipped. --- CLAUDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 2fe4c51a..a75e0c18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,10 @@ Outpost — AI-powered customer support operations platform. See [README](./README.md) for the product overview. +## Code review before push + +Before pushing any non-trivial code change or opening a PR, run `copilotkit-internal:cr-loop` (the CopilotKit-internal 7-agent review-fix loop) on the diff first. This is a standing rule, not case-by-case. If the `pr-review-toolkit` plugin it depends on isn't installed, ask before falling back to a lighter review — don't silently skip it. Exception: mechanical-only diffs (lockfile regen, whitespace) don't need it, per the skill's own scope rules. + ## Community Signals workflow This repo carries a runnable Claude Code skill suite under `.claude/skills/` for the weekly cross-source (Discord + GitHub) community report. The report is produced by Outpost's community manager today via the manual routine; engineering is porting it into Outpost as native TS per [#66](https://github.com/CopilotKit/outpost/issues/66). Until that lands, the skill suite IS the workflow. From 964d6cb0cee14627dec8f4d671585b9fc3fa6012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 15 Jul 2026 06:35:09 -0700 Subject: [PATCH 23/83] fix(sync): unexport MAPPING_CONFIG_KEY to satisfy Next.js route export rules --- apps/web/src/__tests__/sync-api.test.ts | 187 +++++++++++++----- apps/web/src/app/api/sync/force/route.ts | 20 +- apps/web/src/app/api/sync/mappings/route.ts | 33 ++-- apps/worker/src/index.ts | 4 +- .../src/sync/__tests__/status-map.test.ts | 4 +- packages/outpost/shared/src/sync/index.ts | 28 ++- .../outpost/shared/src/sync/status-map.ts | 4 +- 7 files changed, 192 insertions(+), 88 deletions(-) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 6289cfb2..2b0d6fbe 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -171,8 +171,12 @@ describe('POST /api/sync/conflicts/[id]/resolve', () => { mockSyncEventFindUnique.mockResolvedValue({ id: 'se-1', status: 'conflict' }); mockSyncEventUpdate.mockResolvedValue({ id: 'se-1', status: 'success' }); - const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { resolution: 'outpost' }); - const res = await resolveConflict(req as never, { params: Promise.resolve({ id: 'se-1' }) }); + const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { + resolution: 'outpost', + }); + const res = await resolveConflict(req as never, { + params: Promise.resolve({ id: 'se-1' }), + }); const body = await res.json(); expect(body.success).toBe(true); @@ -182,15 +186,23 @@ describe('POST /api/sync/conflicts/[id]/resolve', () => { it('returns 404 for non-existent conflict', async () => { mockSyncEventFindUnique.mockResolvedValue(null); - const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/nope/resolve', { resolution: 'outpost' }); - const res = await resolveConflict(req as never, { params: Promise.resolve({ id: 'nope' }) }); + const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/nope/resolve', { + resolution: 'outpost', + }); + const res = await resolveConflict(req as never, { + params: Promise.resolve({ id: 'nope' }), + }); expect(res.status).toBe(404); }); it('rejects invalid resolution', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { resolution: 'invalid' }); - const res = await resolveConflict(req as never, { params: Promise.resolve({ id: 'se-1' }) }); + const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { + resolution: 'invalid', + }); + const res = await resolveConflict(req as never, { + params: Promise.resolve({ id: 'se-1' }), + }); expect(res.status).toBe(400); }); @@ -198,8 +210,12 @@ describe('POST /api/sync/conflicts/[id]/resolve', () => { it('rejects when event is not a conflict', async () => { mockSyncEventFindUnique.mockResolvedValue({ id: 'se-1', status: 'success' }); - const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { resolution: 'outpost' }); - const res = await resolveConflict(req as never, { params: Promise.resolve({ id: 'se-1' }) }); + const req = makeJsonRequest('http://localhost:3000/api/sync/conflicts/se-1/resolve', { + resolution: 'outpost', + }); + const res = await resolveConflict(req as never, { + params: Promise.resolve({ id: 'se-1' }), + }); expect(res.status).toBe(400); }); @@ -251,7 +267,10 @@ describe('GET /api/sync/mappings', () => { const res = await getMappings(); const body = await res.json(); - expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); + expect(body.statusMappings.linear).toContainEqual({ + externalStatus: 'Triage', + outpostStatus: 'OPEN', + }); expect(body.priorityMappings).toBeDefined(); expect(body.identityMappings).toBeDefined(); expect(body.labelRules).toBeDefined(); @@ -264,7 +283,10 @@ describe('GET /api/sync/mappings', () => { priorityMappings: { linear: [] }, labelRules: { linear: [] }, }; - mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(saved) }); + mockSystemConfigFindUnique.mockResolvedValue({ + key: 'sync.mappingConfig', + value: JSON.stringify(saved), + }); const res = await getMappings(); const body = await res.json(); @@ -274,12 +296,18 @@ describe('GET /api/sync/mappings', () => { it('falls back to defaults when the persisted SystemConfig value is malformed JSON', async () => { mockExternalIdentityFindMany.mockResolvedValue([]); - mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: 'not valid json {{{' }); + mockSystemConfigFindUnique.mockResolvedValue({ + key: 'sync.mappingConfig', + value: 'not valid json {{{', + }); const res = await getMappings(); const body = await res.json(); - expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); + expect(body.statusMappings.linear).toContainEqual({ + externalStatus: 'Triage', + outpostStatus: 'OPEN', + }); expect(body.priorityMappings).toBeDefined(); expect(body.identityMappings).toBeDefined(); expect(body.labelRules).toBeDefined(); @@ -297,7 +325,10 @@ describe('PUT /api/sync/mappings', () => { statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, priorityMappings: { linear: [] }, }; - mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + mockSystemConfigUpsert.mockResolvedValue({ + key: 'sync.mappingConfig', + value: JSON.stringify(config), + }); const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); const res = await putMappings(req as never); @@ -321,10 +352,14 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects statusMappings of the wrong type entirely', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: 'garbage', - priorityMappings: { linear: [] }, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: 'garbage', + priorityMappings: { linear: [] }, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -332,10 +367,14 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects a mapping with an unknown outpostStatus value', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'NOT_REAL' }] }, - priorityMappings: { linear: [] }, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'NOT_REAL' }] }, + priorityMappings: { linear: [] }, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -343,10 +382,16 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects a mapping with an unknown outpostPriority value', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [{ externalPriority: 'X', outpostPriority: 'NOT_REAL' }] }, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'X', outpostPriority: 'NOT_REAL' }], + }, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -356,10 +401,14 @@ describe('PUT /api/sync/mappings', () => { it('requires admin role', async () => { mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(403); @@ -367,10 +416,14 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects completely empty statusMappings/priorityMappings objects instead of silently wiping config', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: {}, - priorityMappings: {}, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: {}, + priorityMappings: {}, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -378,11 +431,15 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects labelRules of the wrong type', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, - labelRules: 'garbage', - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + labelRules: 'garbage', + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -395,7 +452,10 @@ describe('PUT /api/sync/mappings', () => { priorityMappings: { linear: [] }, labelRules: { linear: [{ externalPrefix: 'Priority: ', outpostPrefix: '' }] }, }; - mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + mockSystemConfigUpsert.mockResolvedValue({ + key: 'sync.mappingConfig', + value: JSON.stringify(config), + }); const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); const res = await putMappings(req as never); @@ -411,11 +471,15 @@ describe('PUT /api/sync/mappings', () => { }); it('rejects a completely empty labelRules object instead of silently wiping config', async () => { - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, - labelRules: {}, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + labelRules: {}, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(400); @@ -445,8 +509,16 @@ describe('POST /api/sync/force', () => { it('enqueues status_change and priority_change jobs for every ticket linked to the plugin', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, - { ticketId: 't-2', plugin: 'linear', ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, + { + ticketId: 't-2', + plugin: 'linear', + ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' }, + }, ]); const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); @@ -486,10 +558,17 @@ describe('POST /api/sync/force', () => { it('syncs only the given ticket when ticketId is provided', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, ]); - const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear', ticketId: 't-1' }); + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { + plugin: 'linear', + ticketId: 't-1', + }); const res = await forceSync(req as never); const body = await res.json(); @@ -516,7 +595,11 @@ describe('POST /api/sync/force', () => { mockSyncEventFindFirst.mockResolvedValue(null); mockTicketExternalLinkFindFirst.mockResolvedValue({ id: 'link-1', plugin: 'linear' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, ]); const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); @@ -550,7 +633,11 @@ describe('POST /api/sync/force', () => { it('does not mislabel a mid-loop DB/queue error as "Invalid request body"', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, ]); mockCreateJob.mockRejectedValueOnce(new Error('db down')); diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index d50302e9..02a5801c 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -22,39 +22,27 @@ export async function POST(request: NextRequest) { try { body = await request.json(); } catch { - return NextResponse.json( - { error: 'Invalid request body' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'Invalid request body' }, { status: 400 }); } const plugin = body.plugin; const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; if (!plugin || typeof plugin !== 'string') { - return NextResponse.json( - { error: 'plugin is required' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'plugin is required' }, { status: 400 }); } const [knownPlugin, hasLinks] = await Promise.all([ prisma.syncEvent.findFirst({ where: { - OR: [ - { sourcePlugin: plugin }, - { targetPlugin: plugin }, - ], + OR: [{ sourcePlugin: plugin }, { targetPlugin: plugin }], }, }), prisma.ticketExternalLink.findFirst({ where: { plugin } }), ]); if (!knownPlugin && !hasLinks) { - return NextResponse.json( - { error: `Unknown plugin: ${plugin}` }, - { status: 404 }, - ); + return NextResponse.json({ error: `Unknown plugin: ${plugin}` }, { status: 404 }); } const links = await prisma.ticketExternalLink.findMany({ diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 6674b268..335f1487 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -8,7 +8,10 @@ import { TicketStatus, TicketPriority } from '@copilotkit/outpost/shared'; * to SystemConfig yet (see PUT below, which persists real overrides). * Identity mappings always come live from ExternalIdentity records. */ -const DEFAULT_STATUS_MAPPINGS: Record> = { +const DEFAULT_STATUS_MAPPINGS: Record< + string, + Array<{ externalStatus: string; outpostStatus: string }> +> = { linear: [ { externalStatus: 'Triage', outpostStatus: 'OPEN' }, { externalStatus: 'Backlog', outpostStatus: 'OPEN' }, @@ -23,7 +26,10 @@ const DEFAULT_STATUS_MAPPINGS: Record> = { +const DEFAULT_PRIORITY_MAPPINGS: Record< + string, + Array<{ externalPriority: string; outpostPriority: string }> +> = { linear: [ { externalPriority: '0 (None)', outpostPriority: 'MEDIUM' }, { externalPriority: '1 (Urgent)', outpostPriority: 'CRITICAL' }, @@ -39,7 +45,10 @@ const DEFAULT_PRIORITY_MAPPINGS: Record> = { +const DEFAULT_LABEL_RULES: Record< + string, + Array<{ externalPrefix: string; outpostPrefix: string }> +> = { github: [ { externalPrefix: 'priority:', outpostPrefix: '' }, { externalPrefix: 'type:', outpostPrefix: '' }, @@ -51,7 +60,7 @@ const DEFAULT_LABEL_RULES: Record ({ + const identityMappings = identities.map((ei: (typeof identities)[number]) => ({ id: ei.id, externalPlugin: ei.plugin, externalUserId: ei.externalId, @@ -182,10 +191,7 @@ export async function PUT(request: NextRequest) { try { body = await request.json(); } catch { - return NextResponse.json( - { error: 'Invalid request body' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'Invalid request body' }, { status: 400 }); } if (!body.statusMappings || !body.priorityMappings) { @@ -210,16 +216,15 @@ export async function PUT(request: NextRequest) { } if (body.labelRules !== undefined && !isValidLabelRulesShape(body.labelRules)) { - return NextResponse.json( - { error: 'labelRules has invalid shape' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'labelRules has invalid shape' }, { status: 400 }); } const config: PersistedMappingConfig = { statusMappings: body.statusMappings as PersistedMappingConfig['statusMappings'], priorityMappings: body.priorityMappings as PersistedMappingConfig['priorityMappings'], - ...(body.labelRules ? { labelRules: body.labelRules as PersistedMappingConfig['labelRules'] } : {}), + ...(body.labelRules + ? { labelRules: body.labelRules as PersistedMappingConfig['labelRules'] } + : {}), }; const value = JSON.stringify(config); diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index d7458596..0730f6a3 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -55,8 +55,8 @@ const worker = new Worker({ [JobType.JOB_CLEANUP]: 1, }, jobTimeouts: { - [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow - [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large + [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow + [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts }, }); diff --git a/packages/outpost/shared/src/sync/__tests__/status-map.test.ts b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts index e4f0d0f6..275bd323 100644 --- a/packages/outpost/shared/src/sync/__tests__/status-map.test.ts +++ b/packages/outpost/shared/src/sync/__tests__/status-map.test.ts @@ -37,7 +37,9 @@ describe('loadStatusMap', () => { }); it('falls back to defaults when persisted config has no entry for this plugin', async () => { - const config = { statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] } }; + const config = { + statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] }, + }; const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); const map = await loadStatusMap('github', db); diff --git a/packages/outpost/shared/src/sync/index.ts b/packages/outpost/shared/src/sync/index.ts index cd8feccd..5247bde4 100644 --- a/packages/outpost/shared/src/sync/index.ts +++ b/packages/outpost/shared/src/sync/index.ts @@ -1,7 +1,12 @@ export * from './types.js'; export { SyncEngine } from './engine.js'; export type { SyncEngineDeps } from './engine.js'; -export { StatusMap, createGitHubStatusMap, createLinearStatusMap, loadStatusMap } from './status-map.js'; +export { + StatusMap, + createGitHubStatusMap, + createLinearStatusMap, + loadStatusMap, +} from './status-map.js'; export type { StatusMappingConfig, StatusMapDb } from './status-map.js'; export { PriorityMap, createLinearPriorityMap, createGitHubPriorityMap } from './priority-map.js'; export type { PriorityMappingConfig } from './priority-map.js'; @@ -10,8 +15,18 @@ export type { IdentityMapperDeps } from './identity-map.js'; export { LabelMapper, createGitHubLabelMapper, createLinearLabelMapper } from './label-map.js'; export type { LabelPrefixRule, LabelMapperConfig } from './label-map.js'; export { LinearAdapter, GitHubAdapter } from './adapters/index.js'; -export type { LinearAdapterConfig, LinearClientLike, GitHubAdapterConfig, OctokitLike } from './adapters/index.js'; -export { onTicketCreated, onTicketUpdated, onMessageCreated, registerSyncTriggers } from './triggers.js'; +export type { + LinearAdapterConfig, + LinearClientLike, + GitHubAdapterConfig, + OctokitLike, +} from './adapters/index.js'; +export { + onTicketCreated, + onTicketUpdated, + onMessageCreated, + registerSyncTriggers, +} from './triggers.js'; export type { SyncTicket, SyncMessage, TicketChanges } from './triggers.js'; export { createSyncHooks } from './hooks.js'; export type { SyncHooks } from './hooks.js'; @@ -22,5 +37,10 @@ export { EchoGuard } from './echo-guard.js'; export type { EchoGuardDeps, SyncEventStatus } from './echo-guard.js'; export { ConflictDetector } from './conflict.js'; export type { ConflictDetectorDeps, ConflictInfo } from './conflict.js'; -export { fanoutToGitHub, fanoutStatusChange, fanoutComment, fanoutLabels } from './fanout-github.js'; +export { + fanoutToGitHub, + fanoutStatusChange, + fanoutComment, + fanoutLabels, +} from './fanout-github.js'; export type { GitHubFanoutDeps, GitHubFanoutResult } from './fanout-github.js'; diff --git a/packages/outpost/shared/src/sync/status-map.ts b/packages/outpost/shared/src/sync/status-map.ts index 4a0b9e15..6a5501bd 100644 --- a/packages/outpost/shared/src/sync/status-map.ts +++ b/packages/outpost/shared/src/sync/status-map.ts @@ -93,7 +93,9 @@ const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; /** Minimal Prisma subset needed to load a persisted mapping config. */ export interface StatusMapDb { systemConfig: { - findUnique(args: { where: { key: string } }): Promise<{ key: string; value: string } | null>; + findUnique(args: { + where: { key: string }; + }): Promise<{ key: string; value: string } | null>; }; } From 53353971618866e0b84d5ccca465ca2236952f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 15 Jul 2026 06:54:38 -0700 Subject: [PATCH 24/83] chore(sync): prettier-format planning docs Cosmetic only -- embedded code-fence reflow in the plan/spec docs to satisfy prettier --check. No content change. --- ...mapping-persistence-and-bulk-force-sync.md | 102 ++++++++++++------ ...-persistence-and-bulk-force-sync-design.md | 18 ++-- 2 files changed, 76 insertions(+), 44 deletions(-) diff --git a/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md b/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md index 2757bac3..9843e89c 100644 --- a/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md +++ b/docs/superpowers/plans/2026-07-13-sync-mapping-persistence-and-bulk-force-sync.md @@ -21,10 +21,12 @@ ## Task 1: Persist mapping config via SystemConfig **Files:** + - Modify: `apps/web/src/app/api/sync/mappings/route.ts` - Modify: `apps/web/src/__tests__/sync-api.test.ts:229-272` (the two `describe` blocks for `GET`/`PUT /api/sync/mappings`) **Interfaces:** + - Consumes: `prisma.systemConfig.findUnique({ where: { key } })` / `.upsert({ where, update, create })` — same shape already used in `packages/outpost/shared/src/dispatch/on-call.ts:45-60`. - Produces: `MAPPING_CONFIG_KEY = 'sync.mappingConfig'` constant (exported from this route file) — Task 2 imports the same string literal into `status-map.ts` (kept as a plain string constant, not cross-imported, to avoid a web→shared reverse dependency; both sides must use the exact string `'sync.mappingConfig'`). @@ -46,7 +48,10 @@ describe('GET /api/sync/mappings', () => { const res = await getMappings(); const body = await res.json(); - expect(body.statusMappings.linear).toContainEqual({ externalStatus: 'Triage', outpostStatus: 'OPEN' }); + expect(body.statusMappings.linear).toContainEqual({ + externalStatus: 'Triage', + outpostStatus: 'OPEN', + }); expect(body.priorityMappings).toBeDefined(); expect(body.identityMappings).toBeDefined(); expect(body.labelRules).toBeDefined(); @@ -59,7 +64,10 @@ describe('GET /api/sync/mappings', () => { priorityMappings: { linear: [] }, labelRules: { linear: [] }, }; - mockSystemConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(saved) }); + mockSystemConfigFindUnique.mockResolvedValue({ + key: 'sync.mappingConfig', + value: JSON.stringify(saved), + }); const res = await getMappings(); const body = await res.json(); @@ -79,7 +87,10 @@ describe('PUT /api/sync/mappings', () => { statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, priorityMappings: { linear: [] }, }; - mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + mockSystemConfigUpsert.mockResolvedValue({ + key: 'sync.mappingConfig', + value: JSON.stringify(config), + }); const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', config, 'PUT'); const res = await putMappings(req as never); @@ -105,10 +116,14 @@ describe('PUT /api/sync/mappings', () => { it('requires admin role', async () => { mockGetServerSession.mockResolvedValue(userSession('tm-1', 'MEMBER')); - const req = makeJsonRequest('http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, - }, 'PUT'); + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { linear: [] }, + }, + 'PUT', + ); const res = await putMappings(req as never); expect(res.status).toBe(403); @@ -177,7 +192,7 @@ export async function GET() { include: { member: { select: { id: true, name: true } } }, }); - const identityMappings = identities.map((ei: typeof identities[number]) => ({ + const identityMappings = identities.map((ei: (typeof identities)[number]) => ({ id: ei.id, externalPlugin: ei.plugin, externalUserId: ei.externalId, @@ -231,10 +246,7 @@ export async function PUT(request: NextRequest) { return NextResponse.json(config); } catch { - return NextResponse.json( - { error: 'Invalid request body' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'Invalid request body' }, { status: 400 }); } } ``` @@ -258,11 +270,13 @@ git commit -m "feat(sync): persist mapping config to SystemConfig" ## Task 2: Load status map from persisted config **Files:** + - Modify: `packages/outpost/shared/src/sync/status-map.ts` - Modify: `packages/outpost/shared/src/sync/index.ts:4` (export the new function) - Create: `packages/outpost/shared/src/sync/__tests__/status-map.test.ts` **Interfaces:** + - Consumes: nothing new from earlier tasks (the `'sync.mappingConfig'` key string must match Task 1's `MAPPING_CONFIG_KEY` value exactly). - Produces: `loadStatusMap(plugin: 'linear' | 'github', db: StatusMapDb): Promise` and `export interface StatusMapDb`. Task 3 imports both. @@ -310,7 +324,9 @@ describe('loadStatusMap', () => { }); it('falls back to defaults when persisted config has no entry for this plugin', async () => { - const config = { statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] } }; + const config = { + statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'OPEN' }] }, + }; const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); const map = await loadStatusMap('github', db); @@ -346,7 +362,9 @@ const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; /** Minimal Prisma subset needed to load a persisted mapping config. */ export interface StatusMapDb { systemConfig: { - findUnique(args: { where: { key: string } }): Promise<{ key: string; value: string } | null>; + findUnique(args: { + where: { key: string }; + }): Promise<{ key: string; value: string } | null>; }; } @@ -394,7 +412,12 @@ export async function loadStatusMap( Add the export to `packages/outpost/shared/src/sync/index.ts:4`: ```typescript -export { StatusMap, createGitHubStatusMap, createLinearStatusMap, loadStatusMap } from './status-map.js'; +export { + StatusMap, + createGitHubStatusMap, + createLinearStatusMap, + loadStatusMap, +} from './status-map.js'; export type { StatusMappingConfig, StatusMapDb } from './status-map.js'; ``` @@ -415,10 +438,12 @@ git commit -m "feat(sync): load status map from persisted mapping config" ## Task 3: Let `initializeSyncEngine` accept a pre-loaded status map **Files:** + - Modify: `packages/outpost/shared/src/sync/init.ts` - Create: `packages/outpost/shared/src/sync/__tests__/init.test.ts` **Interfaces:** + - Consumes: `loadStatusMap`, `StatusMapDb` from Task 2 (imported by the caller, not by `init.ts` itself — `init.ts` just accepts an already-built `StatusMap`). - Produces: `InitOptions.statusMapOverride?: StatusMap` — Task 4's `buildSyncEngine` passes this in. @@ -534,11 +559,13 @@ git commit -m "feat(sync): support statusMapOverride in initializeSyncEngine" ## Task 4: Wire the worker to register the Linear adapter **Files:** + - Create: `apps/worker/src/build-sync-engine.ts` - Create: `apps/worker/src/__tests__/build-sync-engine.test.ts` - Modify: `apps/worker/src/index.ts:37-41` **Interfaces:** + - Consumes: `loadStatusMap`, `initializeSyncEngine`, `SyncEngine` from `@copilotkit/outpost/shared` (Tasks 2 & 3); `prisma` from `@copilotkit/outpost/db`; `createJob` from `@copilotkit/outpost/queue`. - Produces: `export async function buildSyncEngine(): Promise` — `index.ts` calls this in place of the bare `new SyncEngine(...)`. @@ -672,10 +699,12 @@ git commit -m "fix(worker): register Linear sync adapter (was never wired up)" ## Task 5: Bulk force-sync **Files:** + - Modify: `apps/web/src/app/api/sync/force/route.ts` - Modify: `apps/web/src/__tests__/sync-api.test.ts:274-306` (the `describe('POST /api/sync/force', ...)` block) **Interfaces:** + - Consumes: `createJob(JobType.TRACKER_SYNC, payload)` from `@copilotkit/outpost/queue` (already mocked in this test file as `mockCreateJob`); `prisma.ticketExternalLink.findMany` (new mock needed). - Produces: nothing consumed by later tasks — this is the last task. @@ -693,8 +722,16 @@ describe('POST /api/sync/force', () => { it('enqueues status_change and priority_change jobs for every ticket linked to the plugin', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, - { ticketId: 't-2', plugin: 'linear', ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, + { + ticketId: 't-2', + plugin: 'linear', + ticket: { id: 't-2', status: 'RESOLVED', priority: 'LOW' }, + }, ]); const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); @@ -734,10 +771,17 @@ describe('POST /api/sync/force', () => { it('syncs only the given ticket when ticketId is provided', async () => { mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); mockTicketExternalLinkFindMany.mockResolvedValue([ - { ticketId: 't-1', plugin: 'linear', ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' } }, + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, ]); - const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear', ticketId: 't-1' }); + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { + plugin: 'linear', + ticketId: 't-1', + }); const res = await forceSync(req as never); const body = await res.json(); @@ -818,26 +862,17 @@ export async function POST(request: NextRequest) { const ticketId = typeof body.ticketId === 'string' ? body.ticketId : undefined; if (!plugin || typeof plugin !== 'string') { - return NextResponse.json( - { error: 'plugin is required' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'plugin is required' }, { status: 400 }); } const knownPlugin = await prisma.syncEvent.findFirst({ where: { - OR: [ - { sourcePlugin: plugin }, - { targetPlugin: plugin }, - ], + OR: [{ sourcePlugin: plugin }, { targetPlugin: plugin }], }, }); if (!knownPlugin) { - return NextResponse.json( - { error: `Unknown plugin: ${plugin}` }, - { status: 404 }, - ); + return NextResponse.json({ error: `Unknown plugin: ${plugin}` }, { status: 404 }); } const links = await prisma.ticketExternalLink.findMany({ @@ -866,10 +901,7 @@ export async function POST(request: NextRequest) { return NextResponse.json({ queued: links.length, jobs }); } catch { - return NextResponse.json( - { error: 'Invalid request body' }, - { status: 400 }, - ); + return NextResponse.json({ error: 'Invalid request body' }, { status: 400 }); } } ``` diff --git a/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md index 4fb5ecb6..e9d02f48 100644 --- a/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md +++ b/docs/superpowers/specs/2026-07-13-sync-mapping-persistence-and-bulk-force-sync-design.md @@ -54,7 +54,7 @@ at worker boot in `apps/worker/src/index.ts`. Add: ```ts // status-map.ts -export async function loadStatusMap(plugin: 'linear' | 'github'): Promise +export async function loadStatusMap(plugin: 'linear' | 'github'): Promise; ``` which reads `SystemConfig["sync.mappingConfig"]`, extracts `statusMappings[plugin]` if @@ -75,14 +75,14 @@ caller, not looked up from a plugin-level map, per `tracker-sync.ts:181-188`). - `ticketId` present → sync that one ticket (existing single-ticket path; still needs the same "current values" push described below, since it's currently also unimplemented). - `ticketId` absent → bulk mode: - 1. `prisma.ticketExternalLink.findMany({ where: { plugin }, include: { ticket: true } })` - 2. For each linked ticket, enqueue two `TRACKER_SYNC` jobs (reusing the existing job - type/handler, untouched) via `createJob`: - - `action: 'status_change'`, `changeData: { status: ticket.status }` - - `action: 'priority_change'`, `changeData: { priority: ticket.priority }` - (`Ticket` has no tags/labels field in the current schema, so `label_change` is not - part of bulk resync — there's no source value to push.) - 3. Return `{ queued: , jobs: }`. + 1. `prisma.ticketExternalLink.findMany({ where: { plugin }, include: { ticket: true } })` + 2. For each linked ticket, enqueue two `TRACKER_SYNC` jobs (reusing the existing job + type/handler, untouched) via `createJob`: + - `action: 'status_change'`, `changeData: { status: ticket.status }` + - `action: 'priority_change'`, `changeData: { priority: ticket.priority }` + (`Ticket` has no tags/labels field in the current schema, so `label_change` is not + part of bulk resync — there's no source value to push.) + 3. Return `{ queued: , jobs: }`. The route only inserts jobs (cheap Postgres writes); the worker performs the actual pushes asynchronously, so this stays fast even for a few hundred linked tickets. No pagination From 93696c75c9e11b23292ab71329babd5d536b9fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 17 Jul 2026 00:02:22 -0700 Subject: [PATCH 25/83] fix(sync): apply persisted priority + label mappings in worker (#96) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /sync dashboard persists statusMappings, priorityMappings, and labelRules, but the worker only loaded the status map — priority/label edits saved + displayed yet had zero effect on sync (init always used the hardcoded createLinearPriorityMap/createLinearLabelMapper). Mirror the status pattern: - loadPriorityMap(plugin, db) in priority-map.ts (+ PriorityMapDb) - loadLabelMapper(plugin, db) in label-map.ts (+ LabelMapperDb) - priorityMapOverride / labelMapperOverride options on initializeSyncEngine - buildSyncEngine loads all three (Promise.all) and passes the overrides - export the loaders + Db types from sync/index Same fallback ladder as loadStatusMap (missing row / malformed JSON / no plugin entry / invalid entries -> hardcoded default). Priority validates the TicketPriority enum; label validates string prefixes. Tests: loadPriorityMap + loadLabelMapper (mirroring loadStatusMap), init priority/label overrides, build-sync-engine wiring. 762 outpost + 1 worker test pass; typecheck clean. Stacked on #95 (sync-mapping-persistence-bulk-force-sync); depends on its loadStatusMap/statusMapOverride pattern. Closes #96 --- .../src/__tests__/build-sync-engine.test.ts | 14 +++- apps/worker/src/build-sync-engine.ts | 18 ++++- .../shared/src/sync/__tests__/init.test.ts | 42 +++++++++- .../src/sync/__tests__/label-map.test.ts | 72 ++++++++++++++++++ .../src/sync/__tests__/priority-map.test.ts | 76 +++++++++++++++++++ packages/outpost/shared/src/sync/index.ts | 18 ++++- packages/outpost/shared/src/sync/init.ts | 12 ++- packages/outpost/shared/src/sync/label-map.ts | 62 +++++++++++++++ .../outpost/shared/src/sync/priority-map.ts | 61 +++++++++++++++ 9 files changed, 363 insertions(+), 12 deletions(-) create mode 100644 packages/outpost/shared/src/sync/__tests__/label-map.test.ts create mode 100644 packages/outpost/shared/src/sync/__tests__/priority-map.test.ts diff --git a/apps/worker/src/__tests__/build-sync-engine.test.ts b/apps/worker/src/__tests__/build-sync-engine.test.ts index 6aaf118a..41ed8053 100644 --- a/apps/worker/src/__tests__/build-sync-engine.test.ts +++ b/apps/worker/src/__tests__/build-sync-engine.test.ts @@ -1,10 +1,14 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const mockLoadStatusMap = vi.fn(); +const mockLoadPriorityMap = vi.fn(); +const mockLoadLabelMapper = vi.fn(); const mockInitializeSyncEngine = vi.fn(); vi.mock('@copilotkit/outpost/shared', () => ({ loadStatusMap: (...args: unknown[]) => mockLoadStatusMap(...args), + loadPriorityMap: (...args: unknown[]) => mockLoadPriorityMap(...args), + loadLabelMapper: (...args: unknown[]) => mockLoadLabelMapper(...args), initializeSyncEngine: (...args: unknown[]) => mockInitializeSyncEngine(...args), })); @@ -25,19 +29,27 @@ describe('buildSyncEngine', () => { vi.clearAllMocks(); }); - it('loads the Linear status map and passes it into initializeSyncEngine', async () => { + it('loads the Linear status/priority/label maps and passes them into initializeSyncEngine', async () => { const fakeStatusMap = { toOutpost: vi.fn() }; + const fakePriorityMap = { toOutpost: vi.fn() }; + const fakeLabelMapper = { toOutpost: vi.fn() }; const fakeEngine = { getPlugin: vi.fn() }; mockLoadStatusMap.mockResolvedValue(fakeStatusMap); + mockLoadPriorityMap.mockResolvedValue(fakePriorityMap); + mockLoadLabelMapper.mockResolvedValue(fakeLabelMapper); mockInitializeSyncEngine.mockReturnValue(fakeEngine); const result = await buildSyncEngine(); expect(mockLoadStatusMap).toHaveBeenCalledWith('linear', prisma); + expect(mockLoadPriorityMap).toHaveBeenCalledWith('linear', prisma); + expect(mockLoadLabelMapper).toHaveBeenCalledWith('linear', prisma); expect(mockInitializeSyncEngine).toHaveBeenCalledWith({ deps: { prisma, createJob }, identityDeps: prisma, statusMapOverride: fakeStatusMap, + priorityMapOverride: fakePriorityMap, + labelMapperOverride: fakeLabelMapper, }); expect(result).toBe(fakeEngine); }); diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index fca1a700..475c0e19 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -12,14 +12,28 @@ import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; -import { loadStatusMap, initializeSyncEngine, type SyncEngine } from '@copilotkit/outpost/shared'; +import { + loadStatusMap, + loadPriorityMap, + loadLabelMapper, + initializeSyncEngine, + type SyncEngine, +} from '@copilotkit/outpost/shared'; export async function buildSyncEngine(): Promise { - const statusMap = await loadStatusMap('linear', prisma as never); + // Load all three persisted mapping configs (status / priority / label), + // each falling back to its hardcoded default when nothing is persisted. + const [statusMap, priorityMap, labelMapper] = await Promise.all([ + loadStatusMap('linear', prisma as never), + loadPriorityMap('linear', prisma as never), + loadLabelMapper('linear', prisma as never), + ]); return initializeSyncEngine({ deps: { prisma: prisma as never, createJob: createJob as never }, identityDeps: prisma as never, statusMapOverride: statusMap, + priorityMapOverride: priorityMap, + labelMapperOverride: labelMapper, }); } diff --git a/packages/outpost/shared/src/sync/__tests__/init.test.ts b/packages/outpost/shared/src/sync/__tests__/init.test.ts index eae591f3..d9c54b7e 100644 --- a/packages/outpost/shared/src/sync/__tests__/init.test.ts +++ b/packages/outpost/shared/src/sync/__tests__/init.test.ts @@ -1,7 +1,9 @@ import { describe, it, expect, vi } from 'vitest'; import { initializeSyncEngine } from '../init.js'; import { StatusMap } from '../status-map.js'; -import { TicketStatus } from '../../types.js'; +import { PriorityMap } from '../priority-map.js'; +import { LabelMapper } from '../label-map.js'; +import { TicketStatus, TicketPriority } from '../../types.js'; function makeIdentityDeps() { return { @@ -66,4 +68,42 @@ describe('initializeSyncEngine', () => { // delegates to its injected StatusMap expect(plugin!.mapStatusToOutpost('Custom')).toBe(TicketStatus.WAITING_ON_TEAM); }); + + it('uses the provided priorityMapOverride instead of the hardcoded default', () => { + const customMap = new PriorityMap({ P0: TicketPriority.CRITICAL }); + + const engine = initializeSyncEngine({ + deps: makeSyncEngineDeps(), + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + priorityMapOverride: customMap, + }); + + const plugin = engine.getPlugin('linear'); + expect(plugin).toBeDefined(); + // mapPriorityToOutpost delegates to the injected PriorityMap. It lives on + // InternalTracker (not the InternalTracker | ExternalTracker union that + // getPlugin returns), so narrow before asserting. + const tracker = plugin as unknown as { + mapPriorityToOutpost(p: string): TicketPriority; + }; + expect(tracker.mapPriorityToOutpost('P0')).toBe(TicketPriority.CRITICAL); + }); + + it('registers the Linear adapter when a labelMapperOverride is supplied', () => { + const customMapper = new LabelMapper({ + rules: [{ externalPrefix: 'X-', outpostPrefix: '' }], + }); + + const engine = initializeSyncEngine({ + deps: makeSyncEngineDeps(), + identityDeps: makeIdentityDeps(), + env: { LINEAR_API_KEY: 'key', LINEAR_TEAM_ID: 'team' }, + labelMapperOverride: customMapper, + }); + + // The label mapper is only exercised via async pushLabels (GraphQL), so + // assert the override is accepted and the adapter still registers. + expect(engine.getPlugin('linear')).toBeDefined(); + }); }); diff --git a/packages/outpost/shared/src/sync/__tests__/label-map.test.ts b/packages/outpost/shared/src/sync/__tests__/label-map.test.ts new file mode 100644 index 00000000..e453b050 --- /dev/null +++ b/packages/outpost/shared/src/sync/__tests__/label-map.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect, vi } from 'vitest'; +import { loadLabelMapper } from '../label-map.js'; + +function makeDb(row: { key: string; value: string } | null) { + return { systemConfig: { findUnique: vi.fn().mockResolvedValue(row) } }; +} + +describe('loadLabelMapper', () => { + it('falls back to the hardcoded Linear mapper when no config row exists', async () => { + const db = makeDb(null); + const mapper = await loadLabelMapper('linear', db); + + // Linear default strips "Priority: " + expect(mapper.toOutpost(['Priority: bug'])).toEqual(['bug']); + }); + + it('falls back to the hardcoded GitHub mapper when no config row exists', async () => { + const db = makeDb(null); + const mapper = await loadLabelMapper('github', db); + + // GitHub default strips "priority:" and excludes "wontfix" + expect(mapper.toOutpost(['priority:high', 'wontfix'])).toEqual(['high']); + }); + + it('builds from persisted rules when present for the requested plugin', async () => { + const config = { + labelRules: { + linear: [{ externalPrefix: 'X-', outpostPrefix: '' }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const mapper = await loadLabelMapper('linear', db); + + expect(mapper.toOutpost(['X-foo'])).toEqual(['foo']); + // The default "Priority: " rule was replaced, so that prefix is no longer stripped + expect(mapper.toOutpost(['Priority: bar'])).toEqual(['Priority: bar']); + }); + + it('falls back to defaults when persisted config has no rules for this plugin', async () => { + const config = { + labelRules: { linear: [{ externalPrefix: 'X-', outpostPrefix: '' }] }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const mapper = await loadLabelMapper('github', db); + + expect(mapper.toOutpost(['priority:high'])).toEqual(['high']); + }); + + it('falls back to defaults when the persisted value is malformed JSON', async () => { + const db = makeDb({ key: 'sync.mappingConfig', value: 'not json' }); + + const mapper = await loadLabelMapper('linear', db); + + expect(mapper.toOutpost(['Priority: bug'])).toEqual(['bug']); + }); + + it('falls back to defaults when all persisted rules have a non-string prefix', async () => { + const config = { + labelRules: { + linear: [{ externalPrefix: 123, outpostPrefix: null }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const mapper = await loadLabelMapper('linear', db); + + // No valid rules → hardcoded default + expect(mapper.toOutpost(['Priority: bug'])).toEqual(['bug']); + }); +}); diff --git a/packages/outpost/shared/src/sync/__tests__/priority-map.test.ts b/packages/outpost/shared/src/sync/__tests__/priority-map.test.ts new file mode 100644 index 00000000..e2ea936c --- /dev/null +++ b/packages/outpost/shared/src/sync/__tests__/priority-map.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, vi } from 'vitest'; +import { loadPriorityMap } from '../priority-map.js'; +import { TicketPriority } from '../../types.js'; + +function makeDb(row: { key: string; value: string } | null) { + return { systemConfig: { findUnique: vi.fn().mockResolvedValue(row) } }; +} + +describe('loadPriorityMap', () => { + it('falls back to the hardcoded Linear map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadPriorityMap('linear', db); + + // Linear default: '1' (Urgent) → CRITICAL + expect(map.toOutpost('1')).toBe(TicketPriority.CRITICAL); + }); + + it('falls back to the hardcoded GitHub map when no config row exists', async () => { + const db = makeDb(null); + const map = await loadPriorityMap('github', db); + + expect(map.toOutpost('high')).toBe(TicketPriority.HIGH); + }); + + it('builds from persisted config when present for the requested plugin', async () => { + const config = { + priorityMappings: { + linear: [{ externalPriority: 'P1', outpostPriority: 'CRITICAL' }], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadPriorityMap('linear', db); + + expect(map.toOutpost('P1')).toBe(TicketPriority.CRITICAL); + // '1' is no longer mapped since the persisted config replaced it entirely + expect(map.toOutpost('1')).toBe(TicketPriority.MEDIUM); // PriorityMap.toOutpost default fallback + }); + + it('falls back to defaults when persisted config has no entry for this plugin', async () => { + const config = { + priorityMappings: { linear: [{ externalPriority: 'P1', outpostPriority: 'HIGH' }] }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadPriorityMap('github', db); + + expect(map.toOutpost('high')).toBe(TicketPriority.HIGH); + }); + + it('falls back to defaults when the persisted value is malformed JSON', async () => { + const db = makeDb({ key: 'sync.mappingConfig', value: 'not json' }); + + const map = await loadPriorityMap('linear', db); + + expect(map.toOutpost('1')).toBe(TicketPriority.CRITICAL); + }); + + it('skips a persisted entry whose outpostPriority is not a valid TicketPriority enum value', async () => { + const config = { + priorityMappings: { + linear: [ + { externalPriority: 'P1', outpostPriority: 'CRITICAL' }, + { externalPriority: 'Pbad', outpostPriority: 'NOT_A_REAL_PRIORITY' }, + ], + }, + }; + const db = makeDb({ key: 'sync.mappingConfig', value: JSON.stringify(config) }); + + const map = await loadPriorityMap('linear', db); + + expect(map.toOutpost('P1')).toBe(TicketPriority.CRITICAL); + // The invalid entry was skipped → falls through to the default MEDIUM + expect(map.toOutpost('Pbad')).toBe(TicketPriority.MEDIUM); + }); +}); diff --git a/packages/outpost/shared/src/sync/index.ts b/packages/outpost/shared/src/sync/index.ts index 5247bde4..058a713f 100644 --- a/packages/outpost/shared/src/sync/index.ts +++ b/packages/outpost/shared/src/sync/index.ts @@ -8,12 +8,22 @@ export { loadStatusMap, } from './status-map.js'; export type { StatusMappingConfig, StatusMapDb } from './status-map.js'; -export { PriorityMap, createLinearPriorityMap, createGitHubPriorityMap } from './priority-map.js'; -export type { PriorityMappingConfig } from './priority-map.js'; +export { + PriorityMap, + createLinearPriorityMap, + createGitHubPriorityMap, + loadPriorityMap, +} from './priority-map.js'; +export type { PriorityMappingConfig, PriorityMapDb } from './priority-map.js'; export { IdentityMapper } from './identity-map.js'; export type { IdentityMapperDeps } from './identity-map.js'; -export { LabelMapper, createGitHubLabelMapper, createLinearLabelMapper } from './label-map.js'; -export type { LabelPrefixRule, LabelMapperConfig } from './label-map.js'; +export { + LabelMapper, + createGitHubLabelMapper, + createLinearLabelMapper, + loadLabelMapper, +} from './label-map.js'; +export type { LabelPrefixRule, LabelMapperConfig, LabelMapperDb } from './label-map.js'; export { LinearAdapter, GitHubAdapter } from './adapters/index.js'; export type { LinearAdapterConfig, diff --git a/packages/outpost/shared/src/sync/init.ts b/packages/outpost/shared/src/sync/init.ts index 2537efa3..74052436 100644 --- a/packages/outpost/shared/src/sync/init.ts +++ b/packages/outpost/shared/src/sync/init.ts @@ -10,8 +10,8 @@ import { SyncEngine } from './engine.js'; import type { SyncEngineDeps } from './engine.js'; import { LinearAdapter } from './adapters/linear.js'; import { createLinearStatusMap, StatusMap } from './status-map.js'; -import { createLinearPriorityMap } from './priority-map.js'; -import { createLinearLabelMapper } from './label-map.js'; +import { createLinearPriorityMap, PriorityMap } from './priority-map.js'; +import { createLinearLabelMapper, LabelMapper } from './label-map.js'; import { IdentityMapper } from './identity-map.js'; import type { IdentityMapperDeps } from './identity-map.js'; @@ -26,6 +26,10 @@ interface InitOptions { env?: Record; /** Pre-built StatusMap to use instead of createLinearStatusMap(). */ statusMapOverride?: StatusMap; + /** Pre-built PriorityMap to use instead of createLinearPriorityMap(). */ + priorityMapOverride?: PriorityMap; + /** Pre-built LabelMapper to use instead of createLinearLabelMapper(). */ + labelMapperOverride?: LabelMapper; } // ─── Initialization ───────────────────────────────────────────────────── @@ -57,8 +61,8 @@ export function initializeSyncEngine(options: InitOptions): SyncEngine { apiKey: linearApiKey, teamId: linearTeamId, statusMap: options.statusMapOverride ?? createLinearStatusMap(), - priorityMap: createLinearPriorityMap(), - labelMapper: createLinearLabelMapper(), + priorityMap: options.priorityMapOverride ?? createLinearPriorityMap(), + labelMapper: options.labelMapperOverride ?? createLinearLabelMapper(), identityMapper, }); engine.registerInternalTracker(adapter); diff --git a/packages/outpost/shared/src/sync/label-map.ts b/packages/outpost/shared/src/sync/label-map.ts index 6017f466..0f55c452 100644 --- a/packages/outpost/shared/src/sync/label-map.ts +++ b/packages/outpost/shared/src/sync/label-map.ts @@ -133,3 +133,65 @@ export function createLinearLabelMapper(): LabelMapper { ], }); } + +// ─── Persisted Config Loading ───────────────────────────────────────────── + +/** Must match the key used by apps/web/src/app/api/sync/mappings/route.ts. */ +const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; + +/** Minimal Prisma subset needed to load a persisted mapping config. */ +export interface LabelMapperDb { + systemConfig: { + findUnique(args: { + where: { key: string }; + }): Promise<{ key: string; value: string } | null>; + }; +} + +interface PersistedLabelRuleEntry { + externalPrefix: string; + outpostPrefix: string; +} + +/** + * Build a LabelMapper for `plugin`, preferring the persisted SystemConfig + * row (written by the /api/sync/mappings dashboard) over the hardcoded + * factory defaults. Falls back to the hardcoded default whenever the + * config row is missing, malformed, or has no rules for this plugin. + * + * Mirrors loadStatusMap in status-map.ts. Note the persisted labelRules + * carry only prefix rules (externalPrefix/outpostPrefix); the `exclude` + * list is not dashboard-editable, so a persisted config produces a mapper + * with no exclusions. + */ +export async function loadLabelMapper( + plugin: 'linear' | 'github', + db: LabelMapperDb, +): Promise { + const fallback = plugin === 'linear' ? createLinearLabelMapper() : createGitHubLabelMapper(); + + const row = await db.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return fallback; + + let parsed: unknown; + try { + parsed = JSON.parse(row.value); + } catch { + return fallback; + } + + const entries = (parsed as { labelRules?: Record }) + ?.labelRules?.[plugin]; + if (!Array.isArray(entries) || entries.length === 0) return fallback; + + const rules: LabelPrefixRule[] = []; + for (const entry of entries) { + if ( + typeof entry?.externalPrefix === 'string' && + typeof entry?.outpostPrefix === 'string' + ) { + rules.push({ externalPrefix: entry.externalPrefix, outpostPrefix: entry.outpostPrefix }); + } + } + return rules.length > 0 ? new LabelMapper({ rules }) : fallback; +} diff --git a/packages/outpost/shared/src/sync/priority-map.ts b/packages/outpost/shared/src/sync/priority-map.ts index d5e8655b..66374a0e 100644 --- a/packages/outpost/shared/src/sync/priority-map.ts +++ b/packages/outpost/shared/src/sync/priority-map.ts @@ -93,3 +93,64 @@ export function createGitHubPriorityMap(): PriorityMap { low: TicketPriority.LOW, }); } + +// ─── Persisted Config Loading ───────────────────────────────────────────── + +/** Must match the key used by apps/web/src/app/api/sync/mappings/route.ts. */ +const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; + +/** Minimal Prisma subset needed to load a persisted mapping config. */ +export interface PriorityMapDb { + systemConfig: { + findUnique(args: { + where: { key: string }; + }): Promise<{ key: string; value: string } | null>; + }; +} + +interface PersistedPriorityMappingEntry { + externalPriority: string; + outpostPriority: TicketPriority; +} + +/** + * Build a PriorityMap for `plugin`, preferring the persisted SystemConfig + * row (written by the /api/sync/mappings dashboard) over the hardcoded + * factory defaults. Falls back to the hardcoded default whenever the + * config row is missing, malformed, or has no entry for this plugin. + * + * Mirrors loadStatusMap in status-map.ts. + */ +export async function loadPriorityMap( + plugin: 'linear' | 'github', + db: PriorityMapDb, +): Promise { + const fallback = plugin === 'linear' ? createLinearPriorityMap() : createGitHubPriorityMap(); + + const row = await db.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); + if (!row) return fallback; + + let parsed: unknown; + try { + parsed = JSON.parse(row.value); + } catch { + return fallback; + } + + const entries = ( + parsed as { priorityMappings?: Record } + )?.priorityMappings?.[plugin]; + if (!Array.isArray(entries) || entries.length === 0) return fallback; + + const config: PriorityMappingConfig = {}; + for (const entry of entries) { + if ( + entry?.externalPriority && + entry?.outpostPriority && + Object.values(TicketPriority).includes(entry.outpostPriority as TicketPriority) + ) { + config[entry.externalPriority] = entry.outpostPriority; + } + } + return Object.keys(config).length > 0 ? new PriorityMap(config) : fallback; +} From 8a1f5165d863e14df3c893e6817333ea3f4fdb0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:10:16 -0400 Subject: [PATCH 26/83] docs(community-signal): Reddit REST, created dates, Notion-only rule, linked reporters, supervisor + loom rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workflow changes to the Weekly Community Signal routine, mirrored in the skills (source of truth) and the Notion Playbook. - Reddit Pulse via Composio REST + a write-scoped API key (Tools=Write in COMPOSIO_API_KEY), not the composio MCP (entity mismatch + read-only keys). - Every card's Source line (and prospect Issue lines) carries `· opened YYYY-MM-DD` (created date, from gh issue view --json createdAt). - Rule: the report lives ONLY in Notion, never in the codebase. Removed commercial-surfaces.json; product-surface-scan diffs against last week's Notion report. Kept reddit-pulse-seen.json as labeled dedup state. - Reporter handles + company badges are hyperlinks on every card; enrich-reporter returns profile_url + company_url. - Supervisor rule: the orchestrator verifies every subagent's output against source and reconciles cross-agent contradictions before publishing. - Loom script spec: <=10 min, dive straight in, walk the report top to bottom (CopilotKit then AG-UI, same section order), explicit no-blame framing. --- .claude/skills/enrich-reporter/SKILL.md | 8 +++- .claude/skills/loom-walkthrough/SKILL.md | 37 +++++++++------- .claude/skills/product-surface-scan/SKILL.md | 37 ++++------------ .claude/skills/slack-tldr/SKILL.md | 2 + .claude/skills/weekly-report/SKILL.md | 36 ++++++++++------ CLAUDE.md | 1 + .../community-signal/commercial-surfaces.json | 43 ------------------- docs/community-signal/reddit-pulse-seen.json | 18 +++++++- 8 files changed, 79 insertions(+), 103 deletions(-) delete mode 100644 docs/community-signal/commercial-surfaces.json diff --git a/.claude/skills/enrich-reporter/SKILL.md b/.claude/skills/enrich-reporter/SKILL.md index 6ad15c46..91906b16 100644 --- a/.claude/skills/enrich-reporter/SKILL.md +++ b/.claude/skills/enrich-reporter/SKILL.md @@ -18,7 +18,10 @@ For each GitHub username below, run: gh api users/ --jq '{login, name, company, bio, blog, twitter_username}' Return a compact one-line-per-user table: - login | name | company | bio | blog | twitter + login | name | company | profile_url | company_url | bio | blog | twitter + + - profile_url = `https://github.com/` (always — used to link the handle on every card). + - company_url = the company's website for enterprise reporters (e.g. Amazon → https://www.amazon.com, Nvidia → https://www.nvidia.com); blank for indie. Used to link the 🏢 Company badge. Don't guess a URL — leave blank if unsure. Then two bulleted lists: - Enterprise reporters (company field populated, OR bio/blog clearly identifies an employer — mark inferred ones as "(inferred)") @@ -32,7 +35,8 @@ Run them in parallel via xargs or a small loop. Under 350 words total. ## Classification rules -- **Direct enterprise:** `company` field populated → use that as canonical affiliation. +- **Direct enterprise (confirmed):** `company` field populated AND corroborated by the bio/blog/a verifiable identity → use that as canonical affiliation, mark `confirmed`. +- **Self-declared only (UNCONFIRMED):** the `company` field names an employer but nothing else corroborates it — no bio mention, no verifiable name/LinkedIn, throwaway-looking account. Still surface it, but mark it **`unconfirmed (self-declared)`** so the report can flag "⚠️ Company unconfirmed" in 🏢 Enterprise. Never present it as fact. (Precedent: `GeauxEric` → `company: Nvidia`, no verifiable identity → unconfirmed.) - **Inferred enterprise:** `company` empty, but bio or blog clearly identifies an employer (e.g. "Engineer @AcmeCorp", LinkedIn profile naming a current role) → label as ` (inferred)`. Still treat as enterprise signal. - **Indie / no affiliation:** no company, no employer clues. Default classification. - **404 / nonexistent user:** note explicitly. Sometimes handles get renamed; check if the issue still resolves. diff --git a/.claude/skills/loom-walkthrough/SKILL.md b/.claude/skills/loom-walkthrough/SKILL.md index b1d1e7cc..4d37fa46 100644 --- a/.claude/skills/loom-walkthrough/SKILL.md +++ b/.claude/skills/loom-walkthrough/SKILL.md @@ -1,6 +1,6 @@ --- name: loom-walkthrough -description: Generate the 5-7 minute Loom walkthrough script for a completed Weekly Community Signal report — a plain spoken briefing that notifies the team of the week's highlights and what the host needs to flag. Straightforward and factual, NOT a radio show or a performance. Runs after every report (invoked by weekly-report) and on "loom script", "walkthrough script", "record the loom", "narrate the report". +description: Generate the ≤10-minute Loom walkthrough script for a completed Weekly Community Signal report — a plain spoken briefing that dives straight into what the team needs to know, walking the report top to bottom (CopilotKit page first, then AG-UI in the same order). Straightforward and factual, NOT a radio show or a performance; no hype opener, no blame framing. Runs after every report (invoked by weekly-report) and on "loom script", "walkthrough script", "record the loom", "narrate the report". --- # Loom walkthrough script @@ -19,7 +19,8 @@ This is an internal report-out. The host is telling the team the week's highligh - **Straightforward and plain.** State the thing, its status, and whether it needs attention. The host is notifying, not narrating a story. - **Still easy to read aloud** — contractions and short sentences are fine (it's spoken, not a memo), but the register is a calm colleague giving an update, not a presenter. -- **Open with the single most important thing** — the biggest highlight or the biggest flag, said plainly. No "hook," no rotating gimmick, no callback bit. +- **Dive straight in — no hype opener.** One short orienting sentence (week + "I'll walk CopilotKit top to bottom, then AG-UI"), then start at the top of the report. No "hook," no rotating gimmick, no callback bit, no "biggest thing first" reordering — the page order IS the order. +- **Never imply the engineering team isn't doing its job.** State status neutrally: an in-progress fix is "a fix is in review" / "in testing," an unowned item "needs an owner assigned," a shipped-broken item is just described by what broke + the fix — never "stalled," "neglected," "dropped the ball," or "how did this ship." Keep every fact; frame it as a status, not a failing. - **No on-camera meta.** Don't narrate the document's structure — no "in this report," "next section," "as you can see." Just say what happened. - **No catchphrase / no through-line slogan.** Don't invent a theme to repeat. If there's a genuine pattern worth naming, state it once, plainly, where it's relevant. - **Numbers spoken, not written.** "about two months," "fifty out of a hundred" — never "~2mo" or "50/100" in the spoken lines. @@ -31,22 +32,28 @@ The Top issues are the core of the briefing. Deliver them as a **numbered list m ## Length -**5–7 minutes** (~800–1100 spoken words). Mark rough time stamps so the host can pace. Always include a "to hit 5 minutes, cut these" note listing the 2–3 most trimmable lines. +**10 minutes or less** (~1,300–1,500 spoken words max; shorter is fine on a quiet week). Mark rough time stamps so the host can pace. Always include a "to trim, cut these" note listing the 2–3 most trimmable lines (usually the Docs beats and the Reddit lines). -## Segment flow (adapt to the week — don't force empty ones) +## Segment flow — walk the report top to bottom -1. **Open** (~15s) — lead with the single most important thing this week (biggest highlight or biggest flag), said plainly. No hook, no gimmick. -2. **The week in one line** — the one-sentence takeaway + anything the team should watch. The TL;DR spoken aloud, plainly. -3. **Top issues — NUMBERED, one beat each** — deliver the report's ranked top issues as a numbered list ("number one … number two …"), each its own short beat with a `[beat]` between, each with its own time marker. Per issue: what it means for a user + status (fixed / fix in progress / not started). Never how it broke internally, and never blur two into one paragraph. (See "Top issues are numbered" above.) -4. **Pain — the high-level read (the CEO segment — slow down here).** NOT issue-by-issue. Name *where people are struggling* as a pattern, per community, plus the one structural pain. The CEO wants the shape of the hurt, not a bug list. (See "Pain segment" below.) -5. **Enterprise** — count + trend + the one-line "how they showed up / what to do." -6. **CopilotKit Reddit** — score + one-phrase vibe (trimmable). -7. **AG-UI — top issues** — same plain treatment; flag the one that matters most. **This is a SEPARATE page** — the report is two pages (main = CopilotKit, sub-page = AG-UI), so open segment 7 with a `[SWITCH to the AG-UI sub-page]` stage cue and a spoken transition that signals the shift ("switching over to AG-UI…") so the viewer knows they've moved to the other page. -8. **AG-UI pain** — where it hurts + the structural constraint (e.g. review bandwidth). -9. **AG-UI Reddit / momentum** — the public win, end the body on an up note. -10. **Close** (~15s) — the week in a sentence, name the 1–2 things the host needs the team to action, point to the linked report. Plain sign-off — no flourish. +**Dive straight in and follow the report's own order, top to bottom — the CopilotKit page first, then the AG-UI sub-page in the same order.** No separate "opener," no hype line, no curated narrative that reorders the page. One short orienting sentence ("This is the community signal for the week of X — I'll walk CopilotKit top to bottom, then AG-UI"), then go. Cover each section in the order it appears on the page; skip a section only when it's empty (say nothing, move on). -**Both pages get airtime — never skip AG-UI.** The routine always produces two pages; even a thin AG-UI week gets segments 7–9 and the `[SWITCH to the AG-UI sub-page]` cue. If AG-UI is genuinely quiet, compress 7–9 into a shorter beat — but cover it and name the page switch. A script that only walks the CopilotKit page is incomplete. +**CopilotKit page (in page order):** +1. **Trends** — one or two sentences: heavy or quiet week, and are we keeping up. Note capped bulk-close sweeps so the resolved number isn't misread. +2. **Top issues — NUMBERED, one beat each** ("number one … number two …"), a `[beat]` between, each with a time marker. Per issue: what it means for a user + status (fixed / fix in review / needs an owner). Never how it broke internally; never blur two into one paragraph. (See "Top issues are numbered.") +3. **Product surface contradictions** (only if present) — the page-vs-page conflicts and whose job it is to reconcile the pages. +4. **Enterprise** — who showed up + the one or two to hand to sales + any enterprise-surface questions. +5. **Demand** — the notable feature asks, briefly (these are asks, not bugs). +6. **Pain — the high-level read.** NOT issue-by-issue. Name *where people are struggling* as a pattern + the one structural pain. The shape of the hurt, not a bug list. (See "Pain segment.") +7. **Docs** — the doc gaps, one line each (trimmable). +8. **Resolved** — what closed this week. +9. **Reddit Pulse** — score + one-phrase vibe (trimmable). + +Then `[SWITCH to the AG-UI sub-page]` with a plain spoken transition ("now the AG-UI page, same walk"), and cover it in the **same page order**: Trends → Top issues (numbered) → Pain (pattern read) → Demand → Docs → Resolved → Reddit Pulse. + +**Close** (~15s) — the week in a sentence, the one or two things the host needs the team to action, point to the linked report. Plain sign-off, no flourish. + +**Both pages get airtime — never skip AG-UI.** Even a thin AG-UI week gets its walk + the `[SWITCH to the AG-UI sub-page]` cue; compress, don't skip. A script that only walks the CopilotKit page is incomplete. ## Pain segment (what the CEO wants) diff --git a/.claude/skills/product-surface-scan/SKILL.md b/.claude/skills/product-surface-scan/SKILL.md index 5064078d..212e6960 100644 --- a/.claude/skills/product-surface-scan/SKILL.md +++ b/.claude/skills/product-surface-scan/SKILL.md @@ -26,8 +26,8 @@ Non-negotiable. When in doubt, fetch again or drop the claim. The orchestrator's 1. **Fetch the canonical pages** (below) with `WebFetch`. If a page 404s or is unreachable, record that (don't guess) — a page going live/dead is itself a signal (e.g. `copilotkit.ai/enterprise` is currently a 404; if it goes live, flag it). 2. **Extract the commercial surfaces + the pricing-tier caps + the free-vs-paid boundaries** from the fetched pages. -3. **Diff against the stored snapshot** (`docs/community-signal/commercial-surfaces.json`) to detect **what changed since last week** — new named product/surface, a new premium feature, a moved free-vs-paid boundary (a cap change), a tier rename/reprice, a page appearing/disappearing. -4. **Return** the surface list + tier table + the "changed since last week" delta + the classifier, and the updated snapshot to write back. +3. **Diff against last week's report in Notion** — read the prior Weekly Community Signal page's 🏢 Enterprise "Surfaces this week" table + the pricing note under it (that IS last week's baseline) to detect **what changed since last week** — new named product/surface, a new premium feature, a moved free-vs-paid boundary (a cap change), a tier rename/reprice, a page appearing/disappearing. **Nothing is stored on disk** — the report lives only in Notion (see the rule in `weekly-report`). +4. **Return** the surface list + tier table + the "changed since last week" delta + the classifier. (Nothing is written to the repo.) ## Canonical pages to scan (re-check every week) @@ -96,7 +96,7 @@ Apply to every issue/thread when deciding whether it belongs in 🏢 Enterprise. ## Detecting "a feature was added" -The diff is the point — leadership wants to know when the commercial product grew. Compare the fresh scan to `docs/community-signal/commercial-surfaces.json` and report any of: +The diff is the point — leadership wants to know when the commercial product grew. Compare the fresh scan to **last week's report in Notion** (its 🏢 Enterprise "Surfaces this week" table + pricing note) and report any of: - **New named surface / product** (a page or a product name that wasn't there last week). - **New premium feature** under an existing surface (e.g. a "coming soon" that shipped, or a new Inspector capability). - **Moved free-vs-paid boundary** — a cap changed (threads/retention/storage/seats), a feature moved between tiers, a price changed. @@ -113,6 +113,8 @@ A second job, for product: **do the scanned pages contradict each other?** Marke - **Every scanned page must be referenced** — cite the URL for each side of a contradiction so the fix target is unambiguous. Link all canonical pages in the output even when they agree (so product has the full reference set). - **A page-vs-page conflict is the target**, but also flag a **page-vs-reality** conflict when a maintainer/GitHub/Discord statement plainly contradicts a page (e.g. a maintainer says a component is "going fully open source" while the product/pricing page still marks it Enterprise) — tag it `page-vs-source` and link both. - **Be under oath — don't invent contradictions.** Quote the exact conflicting text from each page. If two pages merely describe different things, that's not a contradiction. When unsure, describe both statements and mark it `possible`. +- **Map every claim to the actual product model before flagging — two pages describing *different scopes of the same word* is NOT a contradiction.** Read the claim in the context of how the product actually works (fetch `/product` + `/copilotkit-intelligence` + `/pricing` and reconcile them), not as two isolated strings. +- **Known product model — do NOT re-flag (verified live 2026-07-24):** CopilotKit's **Enterprise Intelligence Platform is self-hostable.** Full-platform self-host is on the **Team self-hosted plan / custom Enterprise** (your own Kubernetes, bring-your-own-database, air-gapped supported); the **/pricing** per-tier line "VPC or On-Prem Deployment — Runtime only" on Developer/Pro means only the *runtime* deploys to your infra on those tiers. Those are **different scopes** (full-platform self-host tier vs runtime-deployment location) — the pricing page and the self-hosting doc do **not** contradict each other. Don't file this as a contradiction. - Each finding: ` · Page A: "" () · Page B: "" () · suggested source of truth`. ## Report placement — the ⚠️ Product surface contradictions category @@ -123,31 +125,9 @@ The contradiction check gets its **own category in the report**, and **when ther - **When there are NONE:** don't take top space — render a single quiet line inside the 🏢 Enterprise section: *"Product pages checked for contradictions — none this week."* plus the referenced page list, so the reference set is always present. - This is cross-community (it's about the CopilotKit product), main page only. -## Ledger — `docs/community-signal/commercial-surfaces.json` - -Store one snapshot per run so the next run can diff. Schema: - -```json -{ - "_comment": "Snapshot of CopilotKit commercial surfaces + pricing caps, captured at the start of each weekly report. The next run diffs against the newest entry to detect added/changed commercial features. Keep the last ~8 runs.", - "runs": [ - { - "run_date": "2026-07-10", - "pages_checked": { "pricing": "ok", "copilotkit-intelligence": "ok", "product": "ok", "premium-overview": "ok", "enterprise": "404" }, - "surfaces": ["Enterprise Intelligence Platform", "CopilotKit Cloud", "Self-Hosted Enterprise Intelligence", "Threads & Persistence", "CopilotKit Inspector", "Premium UI components (Fully Headless Chat, Angular SDK)", "Analytics & Self-Learning", "Enterprise security bundle (SOC2/SSO/RBAC)", "Support/SLA", "Slack & Teams integrations"], - "tiers": { - "Developer": { "price": "free", "threads": 200, "retention_days": 3, "storage_gb": 1, "seats": 1 }, - "Pro": { "price": "$39/dev/mo", "threads": 5000, "retention_days": 5, "storage_gb": 10, "seats": 5 }, - "Team": { "price": "$100/dev/mo", "threads": 25000, "retention_days": 14, "storage_gb": 100, "seats": 5, "self_host_with_db": true }, - "Enterprise": { "price": "custom", "threads": "unlimited", "retention_days": "custom", "storage_gb": "custom", "analytics_self_learning": "coming soon" } - }, - "notes": "copilotkit.ai/enterprise 404s; enterprise product lives at /copilotkit-intelligence. Docs MCP had no session — facts via WebFetch." - } - ] -} -``` - -Create the file on the first run if it doesn't exist. Prune to the last ~8 runs. +## No snapshot on disk — Notion is the baseline + +**This scan writes nothing to the repo.** The report and every artifact derived from it live only in Notion (see the "report data lives only in Notion" rule in `weekly-report`). To get the week-over-week diff, read the **prior** Weekly Community Signal page in Notion — its 🏢 Enterprise "Surfaces this week" table + the pricing note under it are last week's baseline — and compare the fresh live scan against that. (Historical note: this used to persist `docs/community-signal/commercial-surfaces.json`; that snapshot was removed — report-derived data does not get committed to the codebase.) ## Output (return to the orchestrator) @@ -159,7 +139,6 @@ Compact, so the orchestrator's context stays small: 4. **Referenced pages** — the full list of scanned page URLs (always returned, so the report carries the reference set even when everything agrees). 5. **Tier caps** — the current free-vs-paid numbers (so the report can judge whether a threads/retention/storage complaint crosses the paid boundary). 6. **Page-reachability notes** — anything that 404'd or was unreachable. -7. **The snapshot JSON** to write into the ledger. ## Cross-references diff --git a/.claude/skills/slack-tldr/SKILL.md b/.claude/skills/slack-tldr/SKILL.md index cf79efc4..391091b6 100644 --- a/.claude/skills/slack-tldr/SKILL.md +++ b/.claude/skills/slack-tldr/SKILL.md @@ -42,6 +42,8 @@ This nested layout replaced the older flat single-line Top-issues bullet + the * - **Title line** `*Weekly Community Signal* 📣` is the first line, then a blank line, then the TL;DR. - **Plain-English TL;DR line** leads — written for a non-engineer reader (marketing, leadership). One sentence: the headline takeaway. Don't pack metrics — bullets handle that. (The dash style is `*TL;DR —* `, not `*TL;DR — *`.) +- **Center it on leadership** — the business-relevant read: production/user impact, fix status ("fix in review"), and enterprise/GTM signal (who's building on us). Not implementation detail. A leader should be able to skim this one line and know what matters this week. +- **No hype — plain and factual.** State what happened; let the reader judge its size. Ban dramatization: no "loud on signal," "broke the front door," "the week's worst bugs," "showed up in force," "silently" as a scare word, etc. Same neutral register as the Loom briefing — describe the items plainly (e.g. "Light week for volume. Worth knowing: a cross-origin auth bug blocks the default useAgent connection; a few bugs fail without surfacing an error; engineers from Microsoft, SAP, AWS, and Nvidia filed issues."). If a phrase sounds like marketing copy, rewrite it flat. - **Top issues are nested per community** — `• 🔝 *Top issues — CopilotKit:*` then each top issue as a ` ◦` sub-bullet, then the same for AG-UI. Short labels (~2–4 words), mirroring that page's Top-issue cards. This replaced the flat one-line version. - **Issues raised** = total combining GitHub + Discord. Don't separate Discord by channel. - **Resolved** = items in `### ✅ Resolved this week`, with a parenthetical short list. diff --git a/.claude/skills/weekly-report/SKILL.md b/.claude/skills/weekly-report/SKILL.md index d6339872..6426e073 100644 --- a/.claude/skills/weekly-report/SKILL.md +++ b/.claude/skills/weekly-report/SKILL.md @@ -25,13 +25,22 @@ The main page is the **CopilotKit** report. It opens with the `## 📦 CopilotKi Covering the **most recent complete Friday→Friday week** (Friday end-date inclusive) for Discord + GitHub. (Reddit Pulse uses a rolling 90-day window — see step 6.) State the window before pulling data. +## The orchestrator verifies every subagent's work (supervisor rule) + +**The orchestrator is accountable for everything published — a subagent's return is INPUT, not truth.** Every subagent (Discord pull, GitHub pull, deep-read, release-scan, enrich-reporter, enrich-prospect, Reddit Pulse, product-surface, link-review, report-sources) can be wrong, stale, or incomplete. Before using any return: + +- **Spot-check its claims against source** — issue/PR numbers + state + dates (`gh`), versions (release-scan), company affiliation (bio, not just the `company` field), links resolve, product-surface quotes appear on the live page. If a claim can't be traced to a source, don't publish it. +- **Reconcile contradictions between subagents** — if two returns disagree (e.g. deep-read says OPEN but release-scan says shipped), run it down before writing. +- **The two formal gates are still mandatory:** the **link-review pass (step 14)** re-verifies every link + product-surface claim, and the **report-sources pass (14b)** defends every placement against evidence and **feeds corrections back into the report** (fix the report first, then the defense reflects it). Loop each until clean. +- **Re-spawn or correct** when a return looks off, rather than passing it through. Precedents this cycle: the Fri→Fri window was set to the wrong week and caught mid-run; enrich flagged a stale ("ex-") employer; the release cross-check caught issues already fixed in a shipped release. None of those should reach the published page. + ## Orchestrator flow 0. **Fresh pull first — before anything else.** `git pull` the repo so you're running the LATEST skills/rules (they're the source of truth and change often — a stale checkout runs an old spec). And pull **fresh** source data for the window from Discord / GitHub / Reddit every run — never reuse a prior run's pull, a cache, or last week's numbers. 1. **Determine the window.** Today's date → most recent complete Fri→Fri. State it. -1b. **Spawn Subagent H — product-surface scan** (see `product-surface-scan` skill). Spawn it **at report start, in parallel with A/B/G**. It fetches CopilotKit's product / pricing / Premium pages and returns: the authoritative **commercial-surface list** (drives the 🏢 Enterprise "Surfaces this week" table), the **free-vs-paid classifier** (used to decide whether each issue belongs in 🏢 Enterprise — a commercial surface — vs Pain/Demand), a **diff of what commercial features changed since last week**, and a **cross-page contradiction check** (page-vs-page / page-vs-source conflicts → the ⚠️ Product surface contradictions category). Writes a snapshot to `docs/community-signal/commercial-surfaces.json`. **Enterprise = CopilotKit's commercial product, NOT "CopilotKit used at a big company"** — apply the classifier, don't shelve a free-OSS bug under Enterprise just because the reporter is enterprise. +1b. **Spawn Subagent H — product-surface scan** (see `product-surface-scan` skill). Spawn it **at report start, in parallel with A/B/G**. It fetches CopilotKit's product / pricing / Premium pages and returns: the authoritative **commercial-surface list** (drives the 🏢 Enterprise "Surfaces this week" table), the **free-vs-paid classifier** (used to decide whether each issue belongs in 🏢 Enterprise — a commercial surface — vs Pain/Demand), a **diff of what commercial features changed since last week**, and a **cross-page contradiction check** (page-vs-page / page-vs-source conflicts → the ⚠️ Product surface contradictions category). It writes nothing to the repo — the week-over-week baseline is last week's report in Notion (see "Report data lives only in Notion" below). **Enterprise = CopilotKit's commercial product, NOT "CopilotKit used at a big company"** — apply the classifier, don't shelve a free-OSS bug under Enterprise just because the reporter is enterprise. 2. **Spawn Subagent A — Discord pull.** Tell it to pull both servers: - CopilotKit (`1122926057641742418`): `#💬｜general` (text `1182553320540352563`) + `#🤔｜support` (forum `1313616713647919218`) @@ -58,12 +67,12 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl 5b. **Spawn Subagent D2 — Deep-enrich prospects** (see `enrich-prospect` skill). AFTER D classifies the enterprise list, take the **prospect shortlist** (recognizable enterprise / well-funded scale-ups, e.g. Jasper AI / commercetools tier) and deep-enrich each: LinkedIn profile (employer verified against the GitHub company — keep searching if mismatched), company website, company size (ARR / latest funding round / employee count). Returns one structured block per prospect for the 🎯 Prospective enterprise customers subsection. Run on the shortlist ONLY, not every reporter. -6. **Subagent E — Reddit Pulse pull** (per-community, rolling 90-day window). Data source = the **`composio`** MCP server (Composio tool router, OAuth). See "Reddit Pulse section" for the full spec; the mechanics: - - **Connection check.** Reddit needs an ACTIVE Composio connection. If `COMPOSIO_SEARCH_TOOLS` reports `has_active_connection: false` for `reddit`, call `COMPOSIO_MANAGE_CONNECTIONS` (toolkit `reddit`, action `add`), surface the returned auth link to Nathan, then `COMPOSIO_WAIT_FOR_CONNECTIONS`. If `composio` isn't connected at all → render "source not configured" and move on. - - **Discover tools once:** `COMPOSIO_SEARCH_TOOLS` (keep the returned `session_id`, reuse it on every later Composio call). +6. **Subagent E — Reddit Pulse pull** (per-community, rolling 90-day window). Data source = **Composio REST + a write-scoped API key** (NOT the `composio` MCP — its OAuth identity can't see the dashboard connection; and a default read-only API key 403s on `tool_execution`). See "Reddit Pulse section" for the full spec; the mechanics: + - **Auth.** Need a Composio API key with the `Tools` resource = **Write**, set as `COMPOSIO_API_KEY` in repo-root `.env`. All calls use header `x-api-key: $COMPOSIO_API_KEY`. + - **Get the connected account.** `GET https://backend.composio.dev/api/v3/connected_accounts?toolkit_slugs=reddit` → pick the `ACTIVE` account's `id` (`ca_…`; it changes whenever the auth config is recreated). If none is ACTIVE → render "source not configured" and move on. (Prior blockers, now avoided: the MCP entity mismatch + a default read-only key.) - **Window:** rolling **last 90 days**. Compute cutoff = now − 90d (epoch seconds); filter posts by `created_utc >= cutoff` client-side (Reddit search has no native date filter). - **Dedup ledger:** load `docs/community-signal/reddit-pulse-seen.json`. Skip any post `id` already listed. After the run, append ALL surfaced + dropped-as-noise ids under a new dated entry (so noise can't resurface), and prune ids whose post date is >90d old (they can't reappear in the window). - - **Execute via `COMPOSIO_MULTI_EXECUTE_TOOL`** (batch independent calls in parallel; large responses save to the Composio sandbox — parse with `COMPOSIO_REMOTE_WORKBENCH`). Tools: + - **Execute via REST:** `POST https://backend.composio.dev/api/v3/tools/execute/` with body `{"connected_account_id":"ca_…","arguments":{…}}`. Response posts nest under `.data.search_results.data.children[].data` (parse defensively — a `posts[]` array may also appear). Tools: - `REDDIT_SEARCH_ACROSS_SUBREDDITS` — one call per `REDDIT_BRAND_TERMS` entry (default `CopilotKit`, `AG-UI`, `ag-ui`), `restrict_sr=false`, `sort` new + relevance. - `REDDIT_RETRIEVE_REDDIT_POST` — per `REDDIT_WATCHLIST` subreddit (default LocalLLaMA, LangChain, AI_Agents, nextjs, SaaS, LLMDevs) for landscape/competitor chatter. - `REDDIT_RETRIEVE_POST_COMMENTS` — for high-signal / debatable threads; pass the **bare base36 article id** (no `t3_`). Top comments are the sentiment. @@ -242,8 +251,8 @@ Every reported item — in 🔝 Top issues, 🔥 Demand, 💢 Pain, and 📚 Doc | Line | Label | Content | |---|---|---| | 1 | **What it is:** | One plain-English line — what the thing actually is, said the way a person would out loud. NOT agent/meta (never "Landed Top issue #1, score 13, mirrored into Enterprise" — placement is obvious from where the card sits). ~8–18 words. Surface experimental/deprecated/pre-release maturity here in plain words if it applies. | - | 2 | **Source:** | The platform + the linked number/thread: `GitHub [#NNNN](url)` or `Discord [thread](url)`. (Mandatory source link.) | - | 3 | **Reported by:** | The reporter's handle, linked, + `🏢 Company` badge if enterprise. | + | 2 | **Source:** | The platform + the linked number/thread + **when it was opened**: `GitHub [#NNNN](url) · opened YYYY-MM-DD` or `Discord [thread](url) · opened YYYY-MM-DD`. (Mandatory source link; the created date is absolute ISO, from `gh issue view --json createdAt` / the thread's first message. On a multi-issue card, list each date, e.g. `· opened 2026-03-23 / 2026-07-21`.) | + | 3 | **Reported by:** | The reporter's handle **linked to their GitHub profile** — `[``login``](https://github.com/login)` — plus a **linked** `🏢 [Company](company-url)` badge if enterprise. Both the profile URL and the company URL come from `enrich-reporter`; never leave the handle or company as plain text. Multiple reporters → link each. | | 4 | **Description:** | The longer, very-readable explanation — 1–3 human sentences, no wall of text, no jargon dump. This is where detail lives (not the one-liner). | | 5 | **CPK version:** | Just the version number — `v1.61.0`, `@copilotkitnext/core 1.54.0`, `unknown`, or `n/a — AG-UI`. **Number only** — the deprecated/experimental note goes in *What it is* / *Description* / *Fix plan*, not here. | | 6 | **Impact:** | Human-readable — who it hits and how bad, in plain terms. (Demand: this is "why it matters".) | @@ -362,8 +371,8 @@ Rules: - **Every post is a source link** to its Reddit permalink. **Sentiment comes from reading the post + top comments** (`REDDIT_RETRIEVE_POST_COMMENTS`), not the title. - **Cross-posts** of the same story merge into one bullet (note the copies + use max engagement). - **Noise** (spam, false-positive keyword hits) is dropped from the section but still recorded in the ledger so it can't resurface. -- **Source-gated:** if the `composio` MCP / Reddit connection isn't available, render "🟠 Reddit Pulse — source not configured this week." and move on — never block the report on it. -- **Data source:** the **`composio`** MCP server (Composio tool router, OAuth — Composio's egress reaches Reddit where this machine's IP is 403-blocked). Connection managed via `COMPOSIO_MANAGE_CONNECTIONS` / `COMPOSIO_WAIT_FOR_CONNECTIONS`; tools discovered via `COMPOSIO_SEARCH_TOOLS` and run via `COMPOSIO_MULTI_EXECUTE_TOOL` (`REDDIT_SEARCH_ACROSS_SUBREDDITS`, `REDDIT_RETRIEVE_REDDIT_POST`, `REDDIT_RETRIEVE_POST_COMMENTS`). Scope vars `REDDIT_BRAND_TERMS` + `REDDIT_WATCHLIST` in the repo-root `.env`. +- **Source-gated:** if there's no write-scoped `COMPOSIO_API_KEY` or no ACTIVE Reddit connected account, render "🟠 Reddit Pulse — source not configured this week." and move on — never block the report on it. +- **Data source:** **Composio REST** (Composio's egress reaches Reddit where this machine's IP is 403-blocked on anonymous reads) — NOT the `composio` MCP (its OAuth identity can't see the dashboard connection) and NOT a default read-only API key (`tool_execution` 403). Use a **write-scoped** Composio API key (`Tools` resource = Write) in `COMPOSIO_API_KEY` (repo-root `.env`); `POST /api/v3/tools/execute/` with the ACTIVE Reddit `connected_account_id` from `GET /api/v3/connected_accounts?toolkit_slugs=reddit`. Tools: `REDDIT_SEARCH_ACROSS_SUBREDDITS`, `REDDIT_RETRIEVE_REDDIT_POST`, `REDDIT_RETRIEVE_POST_COMMENTS`. Scope vars `REDDIT_BRAND_TERMS` + `REDDIT_WATCHLIST` in the repo-root `.env`. ### Reddit Pulse scoring algorithm @@ -406,7 +415,8 @@ Four subsections, in order: **Companies building on us this week** — the signal is **a company currently using/building on us**, surfaced through someone who *currently* works there. - **Verify the current employer** with `gh api users/` AND read the bio — the `company` field is often stale. If the bio says "ex-", "previously", "prior experience: …", they do NOT count. (Precedent: a reporter showed `company: Apple` but bio said "Prior experience: Apple" — ex-Apple, dropped.) - **Ex-employers and "notable individuals" don't count** — track them as community reporters, not enterprise. -- Per-company bullet: who, where they currently work, what they filed, and the strength of signal. +- **Flag unconfirmed employers explicitly.** If the company can't be independently confirmed — only the self-declared GitHub `company` field, no bio / LinkedIn / other corroboration, or the identity itself can't be pinned — append a **⚠️ Company unconfirmed — ** note to that bullet (and mirror it in the 🎯 prospect block). Never present an unverified employer as fact; a reader/sales must see the confidence. (Precedent: `GeauxEric` listed `company: Nvidia` on GitHub with no verifiable name/LinkedIn → bullet marked "Company unconfirmed — self-declared, single-IC.") +- Per-company bullet: who, where they currently work (confirmed or flagged unconfirmed), what they filed, and the strength of signal. - When correcting a prior week's overcount, say so in a short `
` so the trend stays honest. **Enterprise-offering reactions** — explicitly report community reaction to the enterprise surfaces, especially **Slack / Teams integrations** and **threads / persistence**. **If there was no reaction, say so** — silence is itself a signal. @@ -422,7 +432,7 @@ A standing subsection naming **community members who look like enterprise prospe ### 🎯 Prospective enterprise customers {toggle="true"} - **Company:** []() **Name:** []() ← or " — LinkedIn not confirmed" - **Issue:** []() ← use **Source:** []() for Discord/Reddit + **Issue:** []() · opened YYYY-MM-DD ← use **Source:** []() · opened YYYY-MM-DD for Discord/Reddit **Company Details:** **Passed to (sales):** __ ``` @@ -455,7 +465,7 @@ The action checklist. **Draw the items from the `report-sources` evidence pass** - **Every named entity in 🔄 Patterns is hyperlinked** — no bare `#NNNN`, handles, or feature names in Patterns prose. - Forum thread URL: `https://discord.com/channels//` (parent forum channel ID NOT in URL). - Text channel: link to channel + include date. -- GitHub: `[#NNNN](issue-url)` + backtick handle; no profile link unless they have no filed issue. +- GitHub reporters: **link the handle to its GitHub profile** — `[``login``](https://github.com/login)` — and **link the `🏢 Company` badge to the company site** when enterprise (`🏢 [Amazon](https://www.amazon.com)`). Both URLs come from `enrich-reporter` (profile_url + company_url). The issue number itself is linked on the Source line. No plain-text handles or company names on any card. - Append `🏢 ` badge inline next to enterprise users' handles. Indie / solo get no badge. - Identity collisions: merge same person across handles silently in the count; note inline if useful. - Same-author duplicate-filing: one reporter, one signal. @@ -478,7 +488,7 @@ Test before publishing: read each parenthetical aloud and ask "would a non-engin ## Conventions -- **Dated report lists go oldest → newest.** Front-door entries, reporter rosters, Resolved rows, Reddit Pulse threads. +- **Report data lives ONLY in Notion — never in the codebase.** The report and everything derived from it (surface snapshots, issue lists, scored rankings, prospect data, weekly numbers) get published to the Notion pages and nowhere else. **Do not persist report content or per-run snapshots as files in the repo** (no `commercial-surfaces.json`-style ledgers). When a step needs last week's numbers to diff against, read the **prior report in Notion** — that is the baseline. The single allowed data artifact in the repo is `docs/community-signal/reddit-pulse-seen.json`, and only because it is operational **dedup state** (a list of already-seen Reddit post IDs), not report content — its header comment says so. If a future step wants to "save" anything else, the answer is: put it in the Notion report. - **Report list bullets = one sentence.** Deep technical detail lives in 🔄 Patterns or the linked issue. - Don't post to Discord / Reddit — read only. - Don't ping users by handle in Notion; summarize impact instead. diff --git a/CLAUDE.md b/CLAUDE.md index 2fe4c51a..db3b536b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,6 +45,7 @@ Channels mapped per community in the `weekly-report` skill. ### Cross-skill conventions +- **The report lives ONLY in Notion — never document it in the codebase.** Report content and any per-run snapshot/ledger derived from it are published to the Notion pages, not committed as files. Diff-against-last-week reads the prior Notion report, not a stored file. The one allowed repo artifact is `docs/community-signal/reddit-pulse-seen.json` (operational dedup state — seen Reddit post IDs, not report content). - Read-only on Discord. Never post. - Reports are **company-readable** (product, marketing, leadership, sales/CS, engineering) — strip orchestrator process notes. - Convert relative dates to absolute ISO so pages stay interpretable later. diff --git a/docs/community-signal/commercial-surfaces.json b/docs/community-signal/commercial-surfaces.json deleted file mode 100644 index 7f9b6863..00000000 --- a/docs/community-signal/commercial-surfaces.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "_comment": "Snapshot of CopilotKit commercial surfaces + pricing caps, captured at the start of each weekly report by the product-surface-scan skill. The next run diffs against the newest entry to detect added/changed commercial features and cross-page contradictions. Keep the last ~8 runs.", - "runs": [ - { - "run_date": "2026-07-10", - "pages_checked": { - "https://www.copilotkit.ai/pricing": "ok", - "https://www.copilotkit.ai/copilotkit-intelligence": "ok", - "https://www.copilotkit.ai/product": "ok", - "https://www.copilotkit.ai/": "ok", - "https://docs.copilotkit.ai/premium/overview": "ok", - "https://docs.copilotkit.ai/mastra/premium/self-hosting": "ok", - "https://docs.copilotkit.ai/mastra/premium/threads-explained": "ok", - "https://www.copilotkit.ai/enterprise": "404" - }, - "surfaces": [ - "Enterprise Intelligence Platform", - "CopilotKit Cloud (project API key)", - "Self-Hosted Enterprise Intelligence (license key + copilot-intelligence Helm chart)", - "Threads & Persistence", - "CopilotKit Inspector", - "Premium UI components (Fully Headless Chat UI, Angular SDK)", - "Analytics & Self-Learning", - "Enterprise security bundle (SOC2 / SSO / RBAC)", - "Support / SLA", - "Slack & Teams integrations" - ], - "tiers": { - "Developer": { "price": "free", "threads": 200, "retention_days": 3, "storage_gb": 1, "seats": 1, "deployment": "VPC/on-prem runtime only" }, - "Pro": { "price": "$39/dev/mo", "threads": 5000, "retention_days": 5, "storage_gb": 10, "seats": 5, "deployment": "VPC/on-prem runtime only" }, - "Team": { "price": "$500/mo", "threads": 25000, "retention_days": 14, "storage_gb": 100, "seats": 5, "self_host_with_db": true }, - "Enterprise": { "price": "custom", "threads": "unlimited", "retention_days": "custom", "storage_gb": "custom", "analytics_self_learning": "early access" } - }, - "contradictions": [ - { "what": "Angular SDK gated Enterprise (product+pricing) vs maintainer says fully OSS", "type": "page-vs-source", "confidence": "firm", "pages": ["https://www.copilotkit.ai/product", "https://www.copilotkit.ai/pricing"] }, - { "what": "Angular SDK first-class/ungated on homepage vs Enterprise-gated on product+pricing", "type": "page-vs-page", "confidence": "firm", "pages": ["https://www.copilotkit.ai/", "https://www.copilotkit.ai/product", "https://www.copilotkit.ai/pricing"] }, - { "what": "Analytics & Self-Learning 'Early access' (pricing) vs 'Coming Soon' (intelligence+product)", "type": "page-vs-page", "confidence": "firm", "pages": ["https://www.copilotkit.ai/pricing", "https://www.copilotkit.ai/copilotkit-intelligence", "https://www.copilotkit.ai/product"] }, - { "what": "Self-hosting on Developer/Pro (pricing, 'Runtime only') vs Team+/Enterprise only (self-hosting doc)", "type": "page-vs-page", "confidence": "possible", "pages": ["https://www.copilotkit.ai/pricing", "https://docs.copilotkit.ai/mastra/premium/self-hosting"] } - ], - "notes": "copilotkit.ai/enterprise 404s; the enterprise product lives at /copilotkit-intelligence. Docs MCP had no valid session — docs read via WebFetch. Minor: the Intelligence product is named four ways on one page." - } - ] -} diff --git a/docs/community-signal/reddit-pulse-seen.json b/docs/community-signal/reddit-pulse-seen.json index 7eb2dbec..4baefca1 100644 --- a/docs/community-signal/reddit-pulse-seen.json +++ b/docs/community-signal/reddit-pulse-seen.json @@ -1,5 +1,5 @@ { - "_comment": "Reddit Pulse dedup ledger. Each weekly run looks back 90 days but skips any post id already listed here, so a thread is reported once. Add the run's surfaced + noise ids under a dated entry. Prune ids whose created date is older than 90 days from today (they can never reappear in the window anyway).", + "_comment": "DEDUP DATABASE for Reddit Pulse — operational state, NOT the report and NOT the Composio/Reddit connection. The Weekly Community Signal report lives ONLY in Notion; this file is the one allowed data artifact in the repo. It records Reddit post IDs already surfaced in past reports so the rolling 90-day Reddit Pulse never re-reports the same post. Each weekly run: read this file, skip any id listed, then append the run's surfaced + noise ids under a new dated entry; prune ids older than 90 days (they can't reappear in the window). NOTE: connecting to Composio/Reddit is separate and lives in .env (COMPOSIO_API_KEY, write-scoped) + a live connected_account_id fetched at runtime — none of that is stored here.", "runs": [ { "run_date": "2026-06-19", @@ -79,6 +79,22 @@ "1uwjl9x", "1ux8507" ] + }, + { + "run_date": "2026-07-24", + "window": "2026-04-25..2026-07-24", + "seen_ids": [ + "1uy96fy", "1uyoahi", "1uyoxt6", "1uyrx7s", "1uz94e7", + "1v2tif9", "1v3bda8", "1v3cuaq", "1v3dixf", "1v3hism", + "1v3itoj", "1v3v4g6", "1v3v50w", "1v3vkm7", "1v3yhdh", + "1v3yy29", "1v44fci", "1v45xq9", "1v47cou", "1v4ajgy", + "1v4crst", "1v4ebjo", "1v4kh48", "1v4l8na", "1v4mods", + "1v4oarx", "1v4q5nq", "1v4w1kt", "1v4wi3y", "1v4zpxt", + "1v53hk1", "1v54xl5", "1v58gll", "1v59yej", "1v5a92k", + "1v5el2x", "1v5eny9", "1v5eos9", "1v5itwd", "1v5j4eb", + "1v5jv69", "1v5jvx1", "1v5jwce", "1v5jwx2", "1v5jxei", + "1v5k28y", "1v5krfv" + ] } ] } From d98466fb6fe273ba6480ac6c4877ec9891bb4c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:38:00 -0400 Subject: [PATCH 27/83] ci: enable lint, harden secret ignores, correct migration docs Enable lint in CI: - turbo.json declares ESLINT_USE_FLAT_CONFIG under the lint task's env. turbo sanitizes the environment, so 'turbo run lint' failed with "couldn't find eslint.config.js" even with the variable set in the parent shell. The CI job already set the variable; it never reached eslint. - add the Lint step to ci.yml, replacing the skip comment. The required check is named "Lint, Typecheck & Test" but lint never ran. - fix the 4 errors this surfaced: - queue/src/types.ts: three empty interfaces (SlaCheckPayload, JobCleanupPayload, GithubReactionPollPayload) accepted any non-nullish value, defeating payload typing. Now Record, which still accepts {}. - shared/src/platforms/slack.ts: 'blocks as any' cast removed by typing blocks as KnownBlock[] from @slack/web-api. Call-site enumeration: - SlaCheckPayload / JobCleanupPayload / GithubReactionPollPayload: referenced by their handlers and JobPayloadMap in queue/src/types.ts; all call sites pass {} or a typed payload object -> assumptions hold (Record accepts {}). Verified by pnpm typecheck. - blocks (local const, slack.ts postResponse): single push site and one chat.postMessage consumer, both in the same function -> no external call sites. Secret hygiene: .gitignore now covers .env.*, .env.bak*, and .railway-config-pull-*/ while keeping .env.example tracked. An untracked .env.bak holding live Discord/Slack/Anthropic credentials was one 'git add .' from being committed. Docs: deployment.md said to run 'pnpm db:push', but apps/web/start.sh runs 'prisma migrate deploy' on every boot and versioned migrations exist. db push against a migration-managed database drifts the schema and breaks the next migrate deploy. Documents the real mechanism and records the absent backup/restore procedure as a known gap. --- .github/workflows/ci.yml | 10 ++++++-- .gitignore | 10 ++++++++ docs/deployment.md | 23 +++++++++++++++++-- packages/outpost/queue/src/types.ts | 15 +++++------- .../outpost/shared/src/platforms/slack.ts | 5 ++-- turbo.json | 7 +++++- 6 files changed, 54 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd0b4273..f8dee27b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,8 +58,14 @@ jobs: - name: Build run: pnpm build - # Lint skipped: ESLint 9 requires flat config but repo uses legacy .eslintrc.cjs. - # TODO: Migrate to eslint.config.js and re-enable. + # ESLint 9 defaults to flat config while this repo still uses + # .eslintrc.cjs, so the job sets ESLINT_USE_FLAT_CONFIG=false (above) and + # turbo.json declares it under the lint task's `env` — turbo sanitizes the + # environment, so without that declaration this fails with "couldn't find + # eslint.config.js". Migrating to eslint.config.js is still worth doing: + # eslintrc support is deprecated in 9 and removed in 10. + - name: Lint + run: pnpm lint - name: Typecheck run: pnpm typecheck diff --git a/.gitignore b/.gitignore index db3eee6a..14dfe333 100644 --- a/.gitignore +++ b/.gitignore @@ -7,10 +7,20 @@ dist/ out/ # Environment +# Deliberately broad: these files hold live Discord/Slack/Anthropic credentials, and +# a stray `git add .` is the only thing standing between them and a commit. Add new +# allowances explicitly rather than loosening the patterns. .env +.env.* .env.local .env.*.local +.env.bak* +*.env.bak* !.env.example +!.env.*.example + +# Pulled deployment config (may contain resolved secrets) +.railway-config-pull-*/ # Prisma packages/outpost/db/src/generated/ diff --git a/docs/deployment.md b/docs/deployment.md index 59085827..947d553f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -232,9 +232,28 @@ After provisioning PostgreSQL (Railway supports pgvector via `CREATE EXTENSION`) CREATE EXTENSION IF NOT EXISTS vector; ``` -Then run migrations: +The schema is managed by **versioned Prisma migrations** (`packages/outpost/db/prisma/migrations/`), +and `apps/web/start.sh` runs `prisma migrate deploy` on every container start. So a +deployed environment migrates itself — there is no manual step for staging or production. + +To apply migrations by hand (e.g. against a fresh local database): ```bash pnpm db:generate -pnpm db:push +pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma ``` + +> **Do not run `pnpm db:push` against staging or production.** `prisma db push` syncs the +> schema without recording a migration, which puts the database out of step with the +> migration history and makes the next `migrate deploy` fail or clobber changes. It is for +> throwaway local databases and prototyping only. + +To create a new migration during development, use +`prisma migrate dev --name ` and commit the generated directory. + +### Backups + +Railway's managed Postgres handles storage-level durability, but there is **no documented +application-level backup/restore procedure yet** — no scheduled `pg_dump`, and no rehearsed +restore. Treat that as an open gap before relying on this database for anything you cannot +reconstruct. diff --git a/packages/outpost/queue/src/types.ts b/packages/outpost/queue/src/types.ts index 9a08b2c0..fc9ce17a 100644 --- a/packages/outpost/queue/src/types.ts +++ b/packages/outpost/queue/src/types.ts @@ -44,9 +44,8 @@ export interface TicketClassifyPayload { ticketId: string; } -export interface SlaCheckPayload { - // No payload needed — runs against all open tickets -} +/** No payload needed — runs against all open tickets. */ +export type SlaCheckPayload = Record; export interface EscalationPayload { ticketId: string; @@ -79,13 +78,11 @@ export interface TrackerSyncPayload { changeData: Record; } -export interface JobCleanupPayload { - // No payload needed — runs on a fixed schedule -} +/** No payload needed — runs on a fixed schedule. */ +export type JobCleanupPayload = Record; -export interface GithubReactionPollPayload { - // No payload needed — runs against all pending-feedback AI messages -} +/** No payload needed — runs against all pending-feedback AI messages. */ +export type GithubReactionPollPayload = Record; /** Map from JobType to its specific payload shape */ export interface JobPayload { diff --git a/packages/outpost/shared/src/platforms/slack.ts b/packages/outpost/shared/src/platforms/slack.ts index 114d9466..6a4466fd 100644 --- a/packages/outpost/shared/src/platforms/slack.ts +++ b/packages/outpost/shared/src/platforms/slack.ts @@ -6,6 +6,7 @@ */ import { WebClient } from '@slack/web-api'; +import type { KnownBlock } from '@slack/web-api'; import { TicketSource } from '../types.js'; import type { PlatformAdapter, InboundMessage, PlatformUser, FormattedResponse } from './types.js'; @@ -114,7 +115,7 @@ export class SlackAdapter implements PlatformAdapter { const threadTs = extractThreadTs(ticket.sourceId); // Build Block Kit blocks for richer formatting - const blocks: Array> = []; + const blocks: KnownBlock[] = []; // Main response body blocks.push({ @@ -129,7 +130,7 @@ export class SlackAdapter implements PlatformAdapter { channel: ticket.channel, thread_ts: threadTs, text: response.text, // Fallback for notifications - blocks: blocks as any, + blocks, }); return undefined; } diff --git a/turbo.json b/turbo.json index aa60adf1..be096ad2 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,12 @@ "persistent": true }, "lint": { - "dependsOn": ["^build"] + "dependsOn": ["^build"], + // ESLint 9 defaults to flat config; this repo still uses .eslintrc.cjs. + // turbo sanitizes the environment, so the opt-out has to be declared + // here or `turbo run lint` fails with "couldn't find eslint.config.js" + // even when the variable is set in the parent shell. + "env": ["ESLINT_USE_FLAT_CONFIG"] }, "test": { "dependsOn": ["^build"] From 1b7d3add87f4ed08bd9c4ef824a7b74eca906114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:56:56 -0400 Subject: [PATCH 28/83] fix: close CR round-1 bucket (a) findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a4 pnpm lint was broken outside CI. ESLINT_USE_FLAT_CONFIG was set only in the CI job env, so every local 'pnpm lint' died with "couldn't find eslint.config.js". Set it once in the root lint script; turbo passes it through via the lint task's env declaration. Verified with the variable absent from the environment: 10/10 tasks pass. a6 the worker was exempt from the gate this branch adds. Its lint script was "echo 'no eslint config for worker yet'", so the service that owns the SHADOW_MODE post-back path was never linted. Now 'eslint src/', which surfaced two no-explicit-any errors on one line: the SyncEngine construction cast prisma and createJob through 'any'. SyncEngineDeps describes only the slice of Prisma the engine needs with loose Record args, so the real narrower signatures are not assignable — the coercion is deliberate. Assert to SyncEngineDeps['prisma'] and SyncEngineDeps['createJob'] instead, so the cast is tied to a named contract and breaks loudly if that contract changes. a7 turbo served stale lint/typecheck passes after config edits. Added globalDependencies for .eslintrc.cjs, tsconfig.json, vitest.config.ts, .prettierrc. Verified: a content change to .eslintrc.cjs takes the run from 10 cached to 0 cached. a3/a8 secret coverage gaps in .gitignore. .claude/settings.local.json holds SLACK_WEBHOOK_URL_1 but was ignored only by a personal global gitignore, so it was unprotected in every other clone. Added it, plus *.pem/*.key/*.p12/ *.pfx — GITHUB_PRIVATE_KEY arrives as a downloaded .pem. Collapsed the redundant .env.bak patterns into .env.*, keeping .env.example tracked. Verified each path with git check-ignore. a1/a2 the worker health-port docs were inverted. apps/worker/Dockerfile sets ENV HEALTH_PORT=3005 and exposes/probes 3005, so 3005 is the image value, not a local-only override. The worker also resolves PORT ?? HEALTH_PORT ?? 3003, so an injected PORT shadows HEALTH_PORT; the Teams bot reads only HEALTH_PORT. Corrected the table, the note, and the health-check list. a5 migration docs named only web. apps/worker/start.sh:4 also runs 'prisma migrate deploy', so a deploy restarting both gives two concurrent migrators on one database. Documented that, and that Prisma's advisory lock makes the second wait rather than corrupt. Call-site enumeration: - SyncEngineDeps (type-only import, apps/worker/src/index.ts): already exported from packages/outpost/shared/src/sync/index.ts:3, so no new public surface. Used only in the indexed-access positions added here. - SyncEngine constructor: single construction site, unchanged arity and runtime arguments — assertions are type-level only. Verified by pnpm typecheck + pnpm build. - apps/worker lint script: consumed only by 'turbo run lint'; no other caller references it. - root lint script: consumed by CI's Lint step and by developers; CI keeps its own job-level env var, so the two are independent. --- .gitignore | 17 +++++++++++++---- apps/worker/package.json | 2 +- apps/worker/src/index.ts | 11 ++++++++++- docs/deployment.md | 29 +++++++++++++++++++++++------ package.json | 2 +- turbo.json | 3 +++ 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 14dfe333..8377fa21 100644 --- a/.gitignore +++ b/.gitignore @@ -10,15 +10,24 @@ out/ # Deliberately broad: these files hold live Discord/Slack/Anthropic credentials, and # a stray `git add .` is the only thing standing between them and a commit. Add new # allowances explicitly rather than loosening the patterns. +# `.env.*` already covers .env.local, .env.bak., and friends — keep it that way +# instead of accumulating narrower duplicates. .env .env.* -.env.local -.env.*.local -.env.bak* -*.env.bak* !.env.example !.env.*.example +# Credentials and keys. GITHUB_PRIVATE_KEY arrives as a downloaded .pem, which is +# otherwise an ordinary untracked file one `git add .` away from being committed. +*.pem +*.key +*.p12 +*.pfx + +# Agent tool settings — holds SLACK_WEBHOOK_URL_1 and similar. Do not rely on a +# personal global gitignore for this: it protects only the machine that has it. +.claude/settings.local.json + # Pulled deployment config (may contain resolved secrets) .railway-config-pull-*/ diff --git a/apps/worker/package.json b/apps/worker/package.json index ae6eb1a5..8d30fb15 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -9,7 +9,7 @@ "dev": "tsx watch src/index.ts", "start": "node dist/index.js", "typecheck": "tsc --noEmit", - "lint": "echo 'no eslint config for worker yet'", + "lint": "eslint src/", "test": "vitest run" }, "dependencies": { diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 1d96977d..1e8a0c39 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -35,10 +35,19 @@ import { createJob, } from '@copilotkit/outpost/queue'; import { SyncEngine } from '@copilotkit/outpost/shared'; +import type { SyncEngineDeps } from '@copilotkit/outpost/shared'; // ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── -const syncEngine = new SyncEngine({ prisma: prisma as any, createJob: createJob as any }); +// SyncEngineDeps describes only the slice of Prisma the engine needs, using loose +// Record argument shapes. The real PrismaClient and createJob have +// narrower signatures, so they are not assignable in the strict direction — the +// coercion is deliberate. Asserting to the named dep types rather than `any` keeps +// that intent explicit and makes the cast break loudly if SyncEngineDeps changes. +const syncEngine = new SyncEngine({ + prisma: prisma as unknown as SyncEngineDeps['prisma'], + createJob: createJob as SyncEngineDeps['createJob'], +}); const handleTrackerSync = createTrackerSyncHandler(syncEngine); diff --git a/docs/deployment.md b/docs/deployment.md index 947d553f..9e298dcb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -39,12 +39,21 @@ Railway auto-deploys from GitHub and natively supports Docker-based services. | outpost-slack-bot | Worker | 3002 | GET /health | | outpost-teams-bot | Web | 3978 (bot), 3003 (health) | GET /health | | outpost-linear-sync | Web | 3004 | GET /health | -| outpost-worker | Worker | 3003 (3005 locally) | GET /health | +| outpost-worker | Worker | 3005 (image default) | GET /health | | outpost-db | PostgreSQL | -- | -- | Ports are the code's defaults (`process.env.PORT`/`HEALTH_PORT` fallback) — Railway may assign different values via its own `PORT` env var per service. -Note that the worker and the Teams bot both read the same `HEALTH_PORT` variable and both default to `3003`. That is fine on Railway, where each service runs in its own container, but it collides when running them together locally — which is why `.env.example` sets `HEALTH_PORT=3005`. +The worker and the Teams bot both read `HEALTH_PORT`, and both fall back to `3003` when +nothing sets it — which is why `.env.example` sets `HEALTH_PORT=3005`, so the two do not +collide when run together locally. Two details matter in a deployed environment: + +- `apps/worker/Dockerfile` already sets `ENV HEALTH_PORT=3005` and both exposes and probes + `3005`, so the worker image serves health on **3005**, not on the bare code default. +- The worker resolves its port as `PORT ?? HEALTH_PORT ?? 3003` (`apps/worker/src/index.ts`), + so a platform-injected `PORT` **overrides** `HEALTH_PORT`. Setting `HEALTH_PORT` alone will + not move the worker's health port if `PORT` is also present. The Teams bot reads only + `HEALTH_PORT`. ## Environment Variables @@ -222,7 +231,7 @@ All seven services expose health endpoints returning JSON: - Slack bot: `GET /health` (port 3002) - Teams bot: `GET /health` (port 3003) - Linear sync: `GET /health` (port 3004) -- Worker: `GET /health` (port 3003 by default; `HEALTH_PORT=3005` locally to avoid clashing with the Teams bot) +- Worker: `GET /health` (port 3005 — set by `ENV HEALTH_PORT=3005` in its Dockerfile; a platform-injected `PORT` takes precedence over `HEALTH_PORT`) ## Database Setup @@ -232,9 +241,17 @@ After provisioning PostgreSQL (Railway supports pgvector via `CREATE EXTENSION`) CREATE EXTENSION IF NOT EXISTS vector; ``` -The schema is managed by **versioned Prisma migrations** (`packages/outpost/db/prisma/migrations/`), -and `apps/web/start.sh` runs `prisma migrate deploy` on every container start. So a -deployed environment migrates itself — there is no manual step for staging or production. +The schema is managed by **versioned Prisma migrations** (`packages/outpost/db/prisma/migrations/`). +Both `apps/web/start.sh` and `apps/worker/start.sh` run `prisma migrate deploy` on every +container start, so a deployed environment migrates itself — there is no manual step for +staging or production. + +Because two services migrate, a deploy that restarts web and worker together has **two +concurrent migrators** against one database. Prisma takes an advisory lock, so the second +waits rather than corrupting state, but it can fail its startup if the first migration +outlasts the lock timeout — a restart clears it. Worth knowing before adding a third +migrating service, and worth consolidating onto a single migrate step (or a release-phase +job) if migrations grow long. To apply migrations by hand (e.g. against a fresh local database): diff --git a/package.json b/package.json index fda7ebc6..bb1fdeb8 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "lint": "turbo run lint", + "lint": "ESLINT_USE_FLAT_CONFIG=false turbo run lint", "test": "turbo run test", "typecheck": "turbo run typecheck", "db:generate": "turbo run db:generate --filter=@copilotkit/outpost", diff --git a/turbo.json b/turbo.json index be096ad2..0658fa32 100644 --- a/turbo.json +++ b/turbo.json @@ -1,5 +1,8 @@ { "$schema": "https://turbo.build/schema.json", + // Root config files are cache inputs: without them a change to .eslintrc.cjs or + // tsconfig.json leaves turbo serving a cached pass from before the rule change. + "globalDependencies": [".eslintrc.cjs", "tsconfig.json", "vitest.config.ts", ".prettierrc"], "tasks": { "build": { "dependsOn": ["^build"], From 85d98d4b62e5fc3e467cc18f7b050736114d26cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:08:45 -0400 Subject: [PATCH 29/83] fix: close CR round-2 bucket (a) findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate to ESLint flat config (r2a2, r2a3). The previous fix set ESLINT_USE_FLAT_CONFIG only where a shell happened to provide it, so linting worked from the repo root and nowhere else: 'cd apps/worker && pnpm lint' exited 2, editor ESLint integrations failed, and the POSIX inline assignment broke on Windows shells. eslint.config.cjs translates the old .eslintrc.cjs through FlatCompat, so the variable is gone from the root script, the CI job env, and turbo.json, and eslintrc's removal in ESLint 10 stops being a cliff. Verified byte-for-byte behavior: same 0 errors / 37 warnings, same by-rule split (27 no-unused-vars + 10 consistent-type-imports), and per-package lint now passes standalone in apps/worker, apps/discord-bot, and packages/outpost. Deleted .eslintrc.cjs rather than leave two config sources; nothing referenced it and apps/web's 'next lint' is unaffected. Narrow turbo globalDependencies (r2a6). .prettierrc and the root vitest.config.ts feed no turbo task, so listing them invalidated all ten packages' caches on edits that cannot change any task's result. Now eslint.config.cjs + tsconfig.json; verified a content change to the former takes the run from cached to 0 cached. Correct the SHADOW_MODE documentation (r2a4). It claimed the flag only takes effect in the worker and that setting it on a bot 'does nothing'. That is false: apps/discord-bot checks isShadowMode() in thread-create.ts and message-create.ts and diverts to handleShadowThreadCreate / handleShadowMessage, recording the ticket and shadow response silently. There are two independent gates — ingest (bot) and post-back (worker) — and for Discord either alone stops a post. The worker's gate is still the one that covers every platform plus ONBOARDING_DIGEST. Documented as a table so the distinction is not lost again. Correct the HEALTH_PORT rationale (r2a1). The claim that HEALTH_PORT=3005 'keeps them apart' was backwards: worker and teams-bot read the same variable, so one value moves both. Separation comes from each Dockerfile pinning its own (worker 3005, teams-bot 3003). Also noted that an injected PORT moves the worker's listener while its Dockerfile probes 3005 unconditionally. Fixed the same wrong rationale in .env.example. Close remaining .gitignore gaps (r2a5). Added '*.env' (staging.env / prod.env are a different shape than '.env.*'), and made the agent-settings pattern unanchored and suffix-tolerant ('**/settings.local.json', '**/settings.local.json.*') so nested worktrees and .bak copies are covered. Verified eight secret shapes are ignored and .env.example is not. Documented that lint blocks on errors while warnings are reported, and added the missing Prisma verification step to the CI step list. Call-site enumeration: - ESLINT_USE_FLAT_CONFIG: enumerated all three former setters (root package.json lint script, ci.yml job env, turbo.json lint env) — all removed; grep confirms no remaining reference in the repo. - .eslintrc.cjs: grep across json/yml/cjs/ts found no consumer outside eslint.config.cjs's own comments; apps/web has no eslintrc and its 'next lint' still runs (verified). - turbo globalDependencies: consumed only by turbo's cache hashing; the narrowing removes inputs, so no task loses a dependency it needed. --- .env.example | 6 +++-- .eslintrc.cjs | 24 ----------------- .github/workflows/ci.yml | 11 +++----- .gitignore | 10 ++++--- docs/deployment.md | 52 ++++++++++++++++++++++++------------ eslint.config.cjs | 57 ++++++++++++++++++++++++++++++++++++++++ package.json | 2 +- turbo.json | 16 +++++------ 8 files changed, 114 insertions(+), 64 deletions(-) delete mode 100644 .eslintrc.cjs create mode 100644 eslint.config.cjs diff --git a/.env.example b/.env.example index dd159aa3..425fd481 100644 --- a/.env.example +++ b/.env.example @@ -102,8 +102,10 @@ LOG_LEVEL="info" # ─── Worker ────────────────────────────────────────────────────────────────── HEALTH_PORT=3005 # Health check port — read by BOTH the worker and the - # teams-bot (each defaults to 3003, so they collide when - # run together locally; 3005 keeps them apart) + # teams-bot, so one value here moves both (it cannot + # separate them). Deployed images pin their own: + # worker 3005, teams-bot 3003. To run both locally, + # override per process rather than relying on this. # ─── Railway ───────────────────────────────────────────────────────────────── # Railway auto-deploys from GitHub; no deploy hook needed. diff --git a/.eslintrc.cjs b/.eslintrc.cjs deleted file mode 100644 index 966c86fd..00000000 --- a/.eslintrc.cjs +++ /dev/null @@ -1,24 +0,0 @@ -/** @type {import('eslint').Linter.Config} */ -module.exports = { - root: true, - env: { - node: true, - es2022: true, - }, - parser: '@typescript-eslint/parser', - parserOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - }, - plugins: ['@typescript-eslint'], - extends: [ - 'eslint:recommended', - 'plugin:@typescript-eslint/recommended', - ], - rules: { - '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], - '@typescript-eslint/no-explicit-any': 'error', - '@typescript-eslint/consistent-type-imports': 'warn', - }, - ignorePatterns: ['node_modules/', 'dist/', '.next/', '*.js', '!.eslintrc.js'], -}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8dee27b..79c32001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,6 @@ jobs: name: Lint, Typecheck & Test runs-on: ubuntu-latest timeout-minutes: 15 - env: - ESLINT_USE_FLAT_CONFIG: 'false' steps: - name: Checkout @@ -58,12 +56,9 @@ jobs: - name: Build run: pnpm build - # ESLint 9 defaults to flat config while this repo still uses - # .eslintrc.cjs, so the job sets ESLINT_USE_FLAT_CONFIG=false (above) and - # turbo.json declares it under the lint task's `env` — turbo sanitizes the - # environment, so without that declaration this fails with "couldn't find - # eslint.config.js". Migrating to eslint.config.js is still worth doing: - # eslintrc support is deprecated in 9 and removed in 10. + # Config lives in eslint.config.cjs (ESLint 9 flat config), so this needs no + # environment coaxing and behaves the same here, in a package directory, and + # in an editor's ESLint integration. - name: Lint run: pnpm lint diff --git a/.gitignore b/.gitignore index 8377fa21..ca167434 100644 --- a/.gitignore +++ b/.gitignore @@ -10,10 +10,11 @@ out/ # Deliberately broad: these files hold live Discord/Slack/Anthropic credentials, and # a stray `git add .` is the only thing standing between them and a commit. Add new # allowances explicitly rather than loosening the patterns. -# `.env.*` already covers .env.local, .env.bak., and friends — keep it that way -# instead of accumulating narrower duplicates. +# `.env.*` covers .env.local and .env.bak.; `*.env` is a separate shape entirely +# (staging.env, prod.env) that the dot-prefixed patterns do not match. .env .env.* +*.env !.env.example !.env.*.example @@ -26,7 +27,10 @@ out/ # Agent tool settings — holds SLACK_WEBHOOK_URL_1 and similar. Do not rely on a # personal global gitignore for this: it protects only the machine that has it. -.claude/settings.local.json +# Unanchored and suffix-tolerant so nested worktrees and editor .bak/.orig copies are +# covered too — a leaked backup leaks the same secret as the original. +**/settings.local.json +**/settings.local.json.* # Pulled deployment config (may contain resolved secrets) .railway-config-pull-*/ diff --git a/docs/deployment.md b/docs/deployment.md index 9e298dcb..f7b489b2 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -44,16 +44,20 @@ Railway auto-deploys from GitHub and natively supports Docker-based services. Ports are the code's defaults (`process.env.PORT`/`HEALTH_PORT` fallback) — Railway may assign different values via its own `PORT` env var per service. -The worker and the Teams bot both read `HEALTH_PORT`, and both fall back to `3003` when -nothing sets it — which is why `.env.example` sets `HEALTH_PORT=3005`, so the two do not -collide when run together locally. Two details matter in a deployed environment: +The worker and the Teams bot read the **same** `HEALTH_PORT` variable and both fall back to +`3003`. What keeps them apart in a deployed environment is each image pinning its own value: +`apps/worker/Dockerfile` sets `ENV HEALTH_PORT=3005` (exposing and probing 3005), while +`apps/teams-bot/Dockerfile` sets `ENV HEALTH_PORT=3003`. -- `apps/worker/Dockerfile` already sets `ENV HEALTH_PORT=3005` and both exposes and probes - `3005`, so the worker image serves health on **3005**, not on the bare code default. -- The worker resolves its port as `PORT ?? HEALTH_PORT ?? 3003` (`apps/worker/src/index.ts`), - so a platform-injected `PORT` **overrides** `HEALTH_PORT`. Setting `HEALTH_PORT` alone will - not move the worker's health port if `PORT` is also present. The Teams bot reads only - `HEALTH_PORT`. +Because the variable is shared, a single `HEALTH_PORT` in a local `.env` moves **both** +services to that port rather than separating them — running the two together locally needs a +per-process override, not one shared value. (`.env.example`'s `HEALTH_PORT=3005` therefore +suits running the worker alone; it does not by itself resolve a worker + teams-bot clash.) + +One further asymmetry: the worker resolves `PORT ?? HEALTH_PORT ?? 3003` +(`apps/worker/src/index.ts`), so a platform-injected `PORT` **overrides** `HEALTH_PORT` — and +since its Dockerfile probes 3005 unconditionally, an injected `PORT` moves the listener while +the health check keeps checking 3005. The Teams bot reads only `HEALTH_PORT`. ## Environment Variables @@ -142,10 +146,16 @@ The GitHub Actions workflow (`.github/workflows/ci.yml`) runs on every PR and pu 1. Install dependencies (`pnpm install --frozen-lockfile`) 2. Generate Prisma client -3. Build all packages -4. Lint -5. Type check -6. Run tests +3. Verify the Prisma schema and that a migration directory exists +4. Build all packages +5. Lint +6. Type check +7. Run tests + +Lint blocks on ESLint **errors**; warnings are reported without failing the run (37 exist +today, all `no-unused-vars` / `consistent-type-imports`). Bounding warnings to zero means +clearing those first — worth doing, but deliberately not bundled into the change that turned +the step on. ## Environments (staging → production) @@ -172,10 +182,18 @@ response as a shadow `Message` row (`author: outpost-shadow`, `attachments.shado carrying the text it would have posted, plus confidence and latency. Inspect those rows to verify agent behavior without replying to real users. -`SHADOW_MODE` gates both outbound paths: the `AI_RESPONSE` handler (every auto-response -to a user, posted via the platform adapters) and the `ONBOARDING_DIGEST` job, which -posts a daily digest straight to Discord via `DISCORD_DIGEST_CHANNEL_ID` using raw REST. -With shadow mode on, the digest is logged instead of posted. +`SHADOW_MODE` is read by **more than one service**, and each one gates a different point +in the flow. Set it consistently across an environment rather than on a single service: + +| Service | What the flag changes | +| --- | --- | +| `outpost-discord-bot` | At ingest. `thread-create.ts` and `message-create.ts` call `isShadowMode()` and divert to `handleShadowThreadCreate` / `handleShadowMessage`, recording the ticket and a shadow response silently instead of running the normal visible flow (`src/lib/shadow-mode.ts`). | +| `outpost-worker` | At post-back. The `AI_RESPONSE` handler checks the flag immediately before `adapter.postResponse` and persists the response as a shadow `Message` row instead of posting (`queue/src/handlers/ai-response.ts`). Also gates `ONBOARDING_DIGEST`, which posts a daily digest straight to Discord via `DISCORD_DIGEST_CHANNEL_ID` over raw REST. | + +For Discord either gate alone is enough to stop a post, so they are belt-and-braces. The +worker's gate is the one that covers **every** platform (GitHub, Slack, Teams) plus the +digest job, because that is where the adapter call lives — so a staging environment must +have it set on `outpost-worker`, not only on a bot. When adding any new outbound post path, check `SHADOW_MODE` before posting — otherwise staging will deliver to real users regardless of the flag. diff --git a/eslint.config.cjs b/eslint.config.cjs new file mode 100644 index 00000000..f8bfcca2 --- /dev/null +++ b/eslint.config.cjs @@ -0,0 +1,57 @@ +/** + * ESLint flat config. + * + * ESLint 9 looks for this file by default. Before it existed, linting depended on + * ESLINT_USE_FLAT_CONFIG=false to opt back into `.eslintrc.cjs` — which only worked + * where that variable happened to be set, so `pnpm lint` in a package directory, an + * editor's ESLint integration, and any shell that is not POSIX all failed. Defining + * the config here removes the variable from every one of those paths, and removes the + * ESLint 10 deprecation cliff (eslintrc support is dropped there). + * + * The rule set is translated from the previous `.eslintrc.cjs` through FlatCompat so + * behavior is unchanged: same parser, same two extends, same three rule overrides. + * Verified by comparing problem counts before and after the migration. + */ + +const { FlatCompat } = require('@eslint/eslintrc'); +const js = require('@eslint/js'); + +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, +}); + +module.exports = [ + { + // Mirrors the old ignorePatterns. Flat config needs `**/` prefixes to match at + // any depth — a bare `dist/` would only match the repo root. + ignores: [ + '**/node_modules/**', + '**/dist/**', + '**/.next/**', + '**/generated/**', + '**/*.js', + '**/*.cjs', + '**/*.mjs', + ], + }, + ...compat.config({ + root: true, + env: { + node: true, + es2022: true, + }, + parser: '@typescript-eslint/parser', + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + }, + plugins: ['@typescript-eslint'], + extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'], + rules: { + '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/consistent-type-imports': 'warn', + }, + }), +]; diff --git a/package.json b/package.json index bb1fdeb8..fda7ebc6 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "lint": "ESLINT_USE_FLAT_CONFIG=false turbo run lint", + "lint": "turbo run lint", "test": "turbo run test", "typecheck": "turbo run typecheck", "db:generate": "turbo run db:generate --filter=@copilotkit/outpost", diff --git a/turbo.json b/turbo.json index 0658fa32..d638c6c5 100644 --- a/turbo.json +++ b/turbo.json @@ -1,8 +1,11 @@ { "$schema": "https://turbo.build/schema.json", - // Root config files are cache inputs: without them a change to .eslintrc.cjs or - // tsconfig.json leaves turbo serving a cached pass from before the rule change. - "globalDependencies": [".eslintrc.cjs", "tsconfig.json", "vitest.config.ts", ".prettierrc"], + // Root config files that feed the lint and typecheck tasks are cache inputs: + // without them, a change to the ESLint or TS config leaves turbo serving a cached + // pass from before the rule change. Deliberately narrow — listing a file no task + // consumes (.prettierrc, the root vitest.config.ts) would invalidate every + // package's cache on edits that cannot change any task's result. + "globalDependencies": ["eslint.config.cjs", "tsconfig.json"], "tasks": { "build": { "dependsOn": ["^build"], @@ -13,12 +16,7 @@ "persistent": true }, "lint": { - "dependsOn": ["^build"], - // ESLint 9 defaults to flat config; this repo still uses .eslintrc.cjs. - // turbo sanitizes the environment, so the opt-out has to be declared - // here or `turbo run lint` fails with "couldn't find eslint.config.js" - // even when the variable is set in the parent shell. - "env": ["ESLINT_USE_FLAT_CONFIG"] + "dependsOn": ["^build"] }, "test": { "dependsOn": ["^build"] From 46496d2ceb27bc1632485b00a616f900b54cbdfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Sat, 25 Jul 2026 07:07:14 -0400 Subject: [PATCH 30/83] fix: close CR round-3 bucket (a) findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 found seven defects, all in the flat-config migration from round 2. r3a1 declare the config's own dependencies. eslint.config.cjs requires @eslint/eslintrc and @eslint/js; neither was in any package.json, so both resolved only through pnpm's hoisting of eslint's transitives. A hoist setting change or an ESLint 10 bump would have thrown on config load and killed every lint task. Now declared as devDependencies and recorded in the lockfile. r3a4 stop shadowing apps/web. The root flat config took precedence over apps/web/.eslintrc.cjs for direct and editor invocations, and because the Next and react-hooks plugins are not loaded here, 'npx eslint src/...' in apps/web failed with "Definition for rule 'react-hooks/exhaustive-deps' was not found". Added apps/web/** to ignores so web stays owned by next lint and its own eslintrc. Verified: web now reports a plain file-ignored warning instead of a rule error, and next lint still applies its react-hooks / no-img-element rules. This one is worth recording as a verification failure, not just a bug. The round-2 claim of "behavior unchanged" rested on comparing problem counts in packages/outpost — a package whose configuration never changed. That method structurally cannot detect a rule set going missing in a different package. r3a2 the ignore list was never a translation of the old ignorePatterns: **/generated/**, **/*.cjs and **/*.mjs are additions, and apps/web/** now joins them. The comment claiming otherwise was the defect; each entry is now listed with why it is there. r3a3 removed root: true — an eslintrc-only key FlatCompat drops silently, implying a cascade flat config does not have. r3a7 scripts/ and the root vitest.config.ts sit outside every workspace package, so 'turbo run lint' never reached them. Root lint script now lints them directly (0 errors, 4 warnings). r3a5 the documented warning count was wrong: 37 is packages/outpost alone, and apps/web contributes rule families the note did not mention. Replaced the fixed number with the rule families and a pointer to run pnpm lint, since any count written down here goes stale. r3a8 HEALTH_PORT is read by FIVE services (worker, discord-bot, slack-bot, teams-bot, linear-sync), not two. Documented the full table with each fallback and each Dockerfile's pinned value, in deployment.md and .env.example. r3a6 ran prettier over the files this branch authored; docs/deployment.md and package.json were failing prettier --check. Call-site enumeration: - @eslint/eslintrc, @eslint/js: required only by eslint.config.cjs; declared at the root, which is where that file lives. pnpm install --lockfile-only records them in the '.' importer. - eslint.config.cjs ignores: consumed by every eslint invocation. Verified the three affected scopes — packages/outpost (parity 0/37), apps/web (ignored, next lint unaffected), scripts/ (now linted). - root lint script: consumed by CI's Lint step and by developers; the appended 'eslint scripts/ vitest.config.ts' runs after the turbo pass, so a workspace failure still short-circuits first. --- .env.example | 11 +++++----- docs/deployment.md | 55 +++++++++++++++++++++++++++------------------- eslint.config.cjs | 37 +++++++++++++++++++++---------- package.json | 7 +++--- pnpm-lock.yaml | 6 +++++ 5 files changed, 74 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index 425fd481..6b5b612e 100644 --- a/.env.example +++ b/.env.example @@ -101,11 +101,12 @@ NEXT_PUBLIC_SENTRY_DSN="" LOG_LEVEL="info" # ─── Worker ────────────────────────────────────────────────────────────────── -HEALTH_PORT=3005 # Health check port — read by BOTH the worker and the - # teams-bot, so one value here moves both (it cannot - # separate them). Deployed images pin their own: - # worker 3005, teams-bot 3003. To run both locally, - # override per process rather than relying on this. +HEALTH_PORT=3005 # Health check port — read by FIVE services (worker, + # discord-bot, slack-bot, teams-bot, linear-sync), so one + # value here moves all of them onto the same port; it + # cannot separate them. Deployed images pin their own + # (discord 3001, slack 3002, teams 3003, worker 3005). + # Running several locally needs a per-process override. # ─── Railway ───────────────────────────────────────────────────────────────── # Railway auto-deploys from GitHub; no deploy hook needed. diff --git a/docs/deployment.md b/docs/deployment.md index f7b489b2..285e05d9 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -44,15 +44,20 @@ Railway auto-deploys from GitHub and natively supports Docker-based services. Ports are the code's defaults (`process.env.PORT`/`HEALTH_PORT` fallback) — Railway may assign different values via its own `PORT` env var per service. -The worker and the Teams bot read the **same** `HEALTH_PORT` variable and both fall back to -`3003`. What keeps them apart in a deployed environment is each image pinning its own value: -`apps/worker/Dockerfile` sets `ENV HEALTH_PORT=3005` (exposing and probing 3005), while -`apps/teams-bot/Dockerfile` sets `ENV HEALTH_PORT=3003`. +**Five** services read the same `HEALTH_PORT` variable, each with a different fallback: -Because the variable is shared, a single `HEALTH_PORT` in a local `.env` moves **both** -services to that port rather than separating them — running the two together locally needs a -per-process override, not one shared value. (`.env.example`'s `HEALTH_PORT=3005` therefore -suits running the worker alone; it does not by itself resolve a worker + teams-bot clash.) +| Service | Fallback in code | Pinned by its Dockerfile | +| --------------------- | ---------------- | ------------------------ | +| `outpost-discord-bot` | 3001 | `ENV HEALTH_PORT=3001` | +| `outpost-slack-bot` | 3002 | `ENV HEALTH_PORT=3002` | +| `outpost-teams-bot` | 3003 | `ENV HEALTH_PORT=3003` | +| `outpost-linear-sync` | 3004 | (not pinned) | +| `outpost-worker` | 3003 | `ENV HEALTH_PORT=3005` | + +What separates them in a deployed environment is each image pinning its own value — not the +variable itself. So a single `HEALTH_PORT` in a shared local `.env` collapses **all five** +onto that one port rather than separating anything: `.env.example`'s `HEALTH_PORT=3005` suits +running one service at a time, and running several together needs a per-process override. One further asymmetry: the worker resolves `PORT ?? HEALTH_PORT ?? 3003` (`apps/worker/src/index.ts`), so a platform-injected `PORT` **overrides** `HEALTH_PORT` — and @@ -152,10 +157,16 @@ The GitHub Actions workflow (`.github/workflows/ci.yml`) runs on every PR and pu 6. Type check 7. Run tests -Lint blocks on ESLint **errors**; warnings are reported without failing the run (37 exist -today, all `no-unused-vars` / `consistent-type-imports`). Bounding warnings to zero means -clearing those first — worth doing, but deliberately not bundled into the change that turned -the step on. +Lint blocks on ESLint **errors**; warnings are reported without failing the run. A backlog of +warnings exists across several packages (`no-unused-vars` and `consistent-type-imports` in the +shared packages, plus `react-hooks/exhaustive-deps` and `no-img-element` in `apps/web`) — run +`pnpm lint` for the current list rather than trusting a number written down here. Bounding +warnings to zero means clearing that backlog first: worth doing, but deliberately not bundled +into the change that turned the step on. + +`apps/web` is linted by `next lint` against its own `apps/web/.eslintrc.cjs`; every other +workspace uses the root `eslint.config.cjs`. Migrating web is outstanding — `next lint` is +removed in Next 16. ## Environments (staging → production) @@ -163,12 +174,12 @@ Railway hosts two environments in the `outpost` project, each with its **own** P `main` is the known-good release line: it is what production runs. Development work — features, fixes, chores — happens on branches, which merge into `staging` for integration testing. Nothing reaches `main` until it has soaked on staging. -| | staging | production | -| --- | --- | --- | -| Deploys from | `staging` branch (CI-gated) | `main` (CI-gated) | -| Web URL | `outpost-web-staging.up.railway.app` | `outpost.copilotkit.ai` | -| Database | own Postgres (isolated) | own Postgres | -| Role | integration / soak | known good | +| | staging | production | +| ------------ | ------------------------------------ | ----------------------- | +| Deploys from | `staging` branch (CI-gated) | `main` (CI-gated) | +| Web URL | `outpost-web-staging.up.railway.app` | `outpost.copilotkit.ai` | +| Database | own Postgres (isolated) | own Postgres | +| Role | integration / soak | known good | Four services carry deploy triggers in both environments: `outpost-web`, `outpost-github-app`, `outpost-discord-bot`, `outpost-worker`. The remaining three (`outpost-slack-bot`, `outpost-teams-bot`, `outpost-linear-sync`) are optional integrations — deployed manually / left offline until their credentials are configured. @@ -185,10 +196,10 @@ to verify agent behavior without replying to real users. `SHADOW_MODE` is read by **more than one service**, and each one gates a different point in the flow. Set it consistently across an environment rather than on a single service: -| Service | What the flag changes | -| --- | --- | -| `outpost-discord-bot` | At ingest. `thread-create.ts` and `message-create.ts` call `isShadowMode()` and divert to `handleShadowThreadCreate` / `handleShadowMessage`, recording the ticket and a shadow response silently instead of running the normal visible flow (`src/lib/shadow-mode.ts`). | -| `outpost-worker` | At post-back. The `AI_RESPONSE` handler checks the flag immediately before `adapter.postResponse` and persists the response as a shadow `Message` row instead of posting (`queue/src/handlers/ai-response.ts`). Also gates `ONBOARDING_DIGEST`, which posts a daily digest straight to Discord via `DISCORD_DIGEST_CHANNEL_ID` over raw REST. | +| Service | What the flag changes | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `outpost-discord-bot` | At ingest. `thread-create.ts` and `message-create.ts` call `isShadowMode()` and divert to `handleShadowThreadCreate` / `handleShadowMessage`, recording the ticket and a shadow response silently instead of running the normal visible flow (`src/lib/shadow-mode.ts`). | +| `outpost-worker` | At post-back. The `AI_RESPONSE` handler checks the flag immediately before `adapter.postResponse` and persists the response as a shadow `Message` row instead of posting (`queue/src/handlers/ai-response.ts`). Also gates `ONBOARDING_DIGEST`, which posts a daily digest straight to Discord via `DISCORD_DIGEST_CHANNEL_ID` over raw REST. | For Discord either gate alone is enough to stop a post, so they are belt-and-braces. The worker's gate is the one that covers **every** platform (GitHub, Slack, Teams) plus the diff --git a/eslint.config.cjs b/eslint.config.cjs index f8bfcca2..db630fea 100644 --- a/eslint.config.cjs +++ b/eslint.config.cjs @@ -1,16 +1,21 @@ /** - * ESLint flat config. + * ESLint flat config for every workspace except `apps/web`. * * ESLint 9 looks for this file by default. Before it existed, linting depended on - * ESLINT_USE_FLAT_CONFIG=false to opt back into `.eslintrc.cjs` — which only worked - * where that variable happened to be set, so `pnpm lint` in a package directory, an - * editor's ESLint integration, and any shell that is not POSIX all failed. Defining - * the config here removes the variable from every one of those paths, and removes the - * ESLint 10 deprecation cliff (eslintrc support is dropped there). + * ESLINT_USE_FLAT_CONFIG=false to opt back into `.eslintrc.cjs` — which only worked where + * that variable happened to be set, so `pnpm lint` inside a package, an editor's ESLint + * integration, and any non-POSIX shell all failed. Defining the config here removes the + * variable from all of those paths. * - * The rule set is translated from the previous `.eslintrc.cjs` through FlatCompat so - * behavior is unchanged: same parser, same two extends, same three rule overrides. - * Verified by comparing problem counts before and after the migration. + * Translated from the former root `.eslintrc.cjs` via FlatCompat: same parser, same two + * extends, same three rule overrides. The ignore list is NOT a literal copy — see below. + * + * `apps/web` is deliberately excluded. It keeps its own `.eslintrc.cjs` (Next.js rules, + * including react-hooks) and is linted by `next lint` through its own package script. If + * this config applied there, it would shadow that eslintrc and ESLint would fail with + * "Definition for rule 'react-hooks/exhaustive-deps' was not found", since the Next and + * react-hooks plugins are not loaded here. Migrating web is still outstanding: `next lint` + * is removed in Next 16, and until then eslintrc lives on in that one package. */ const { FlatCompat } = require('@eslint/eslintrc'); @@ -23,8 +28,14 @@ const compat = new FlatCompat({ module.exports = [ { - // Mirrors the old ignorePatterns. Flat config needs `**/` prefixes to match at - // any depth — a bare `dist/` would only match the repo root. + // The former eslintrc ignored: node_modules/, dist/, .next/, *.js. + // Flat config needs `**/` to match at any depth — a bare `dist/` matches only the + // repo root. Three entries are additions, not translations: + // - apps/web/** — owned by next lint + apps/web/.eslintrc.cjs (see above) + // - **/generated/** — Prisma client output, not hand-written source + // - **/*.cjs, **/*.mjs — config files (this one, postcss.config.cjs); the old + // config ignored only `*.js`, so these were nominally in + // scope. Excluding them is a deliberate scope reduction. ignores: [ '**/node_modules/**', '**/dist/**', @@ -33,10 +44,12 @@ module.exports = [ '**/*.js', '**/*.cjs', '**/*.mjs', + 'apps/web/**', ], }, ...compat.config({ - root: true, + // NOTE: no `root: true` here — that key is eslintrc-only and FlatCompat drops it + // silently. Flat config has no cascade, so there is nothing to root. env: { node: true, es2022: true, diff --git a/package.json b/package.json index fda7ebc6..e72e7209 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "lint": "turbo run lint", + "lint": "turbo run lint && eslint scripts/ vitest.config.ts", "test": "turbo run test", "typecheck": "turbo run typecheck", "db:generate": "turbo run db:generate --filter=@copilotkit/outpost", @@ -16,6 +16,8 @@ "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"" }, "devDependencies": { + "@eslint/eslintrc": "^3.3.5", + "@eslint/js": "^9.39.4", "@linear/sdk": "^81.0.0", "@typescript-eslint/eslint-plugin": "^8.58.2", "@typescript-eslint/parser": "^8.58.2", @@ -30,6 +32,5 @@ "engines": { "node": ">=20.0.0" }, - "dependencies": { - } + "dependencies": {} } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b3a28953..63420a32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,12 @@ importers: .: devDependencies: + '@eslint/eslintrc': + specifier: ^3.3.5 + version: 3.3.5 + '@eslint/js': + specifier: ^9.39.4 + version: 9.39.4 '@linear/sdk': specifier: ^81.0.0 version: 81.0.0(graphql@16.13.2) From f9ad8bd0a193f99b0d928712d143dd50491a39ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:51:13 -0400 Subject: [PATCH 31/83] chore: split lint enablement out of this branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four CR rounds produced 28 bucket (a) findings; the large majority were edges of one change — enabling ESLint in CI. That change alters lint resolution for every package, for editors, for per-package invocation, and for apps/web's separate Next config, and each round surfaced another scope it did or did not cover. It is being landed separately so those effects can be verified in isolation rather than riding along with unrelated fixes. Reverted from this branch (moves to the lint PR): - eslint.config.cjs and the deletion of .eslintrc.cjs - the CI Lint step (replaced with a comment stating why the required check named "Lint, Typecheck & Test" does not yet run lint, so the gap is documented rather than silent) - @eslint/eslintrc + @eslint/js devDependencies - apps/worker's eslint script, back to its previous placeholder - the root lint script's extra 'eslint scripts/ vitest.config.ts' pass Kept — each verified independently of lint: - .gitignore: .claude/settings.local.json was protected only by a personal global gitignore, so every other clone could commit the SLACK_WEBHOOK_URL_1 it holds. Added it plus *.pem/*.key/*.p12/*.pfx (GITHUB_PRIVATE_KEY arrives as a downloaded .pem) and *.env, which .env.* does not match. Verified with git check-ignore across eight secret shapes, and that .env.example stays tracked. - SHADOW_MODE documented correctly in BOTH deployment.md and .env.example. The previous wording claimed the flag only takes effect in the worker and that setting it on a bot 'does nothing'. apps/discord-bot checks isShadowMode() at ingest and diverts to the silent shadow path, so there are two independent gates. The .env.example copy of that error was missed when deployment.md was corrected; both now describe the same behavior. - HEALTH_PORT: five services read it, each with its own fallback and its own Dockerfile pin, so one shared value collapses them rather than separating them. Also documented that PORT and HEALTH_PORT are different listeners for the Teams bot and Linear sync, not a fallback chain as they are for the worker, and corrected the worker's port from 3003 to 3005. - Migrations: both web and worker run 'prisma migrate deploy' on start, so a joint restart has two concurrent migrators; replaced the instruction to run 'pnpm db:push' against a migration-managed database, and recorded the absent backup/restore procedure as a known gap. - types.ts: three empty payload interfaces accepted any non-nullish value; now Record. - slack.ts: 'blocks as any' replaced with KnownBlock[] from @slack/web-api. - worker: SyncEngine construction asserts to SyncEngineDeps['prisma'] and ['createJob'] instead of any, so the deliberate structural coercion is tied to a named contract. - turbo.json: .eslintrc.cjs and tsconfig.json declared as cache inputs, so a config change no longer serves a cached pass. Verified at this commit: typecheck 10/10, tests 10/10, build 10/10. --- .env.example | 15 ++++++--- .eslintrc.cjs | 24 ++++++++++++++ .github/workflows/ci.yml | 13 +++++--- apps/worker/package.json | 2 +- docs/deployment.md | 7 +++- eslint.config.cjs | 70 ---------------------------------------- package.json | 7 ++-- pnpm-lock.yaml | 6 ---- turbo.json | 2 +- 9 files changed, 53 insertions(+), 93 deletions(-) create mode 100644 .eslintrc.cjs delete mode 100644 eslint.config.cjs diff --git a/.env.example b/.env.example index 6b5b612e..0c06ed72 100644 --- a/.env.example +++ b/.env.example @@ -31,11 +31,16 @@ AI_SENTIMENT_MODEL= # Override sentiment analysis model # ─── Shadow Mode ───────────────────────────────────────────────────────────── # Set to 'true' to run the full AI pipeline but LOG responses instead of posting -# them to Discord/GitHub/Slack/Teams. The gate lives in the AI_RESPONSE queue -# handler, which runs in the WORKER service — so SHADOW_MODE MUST be set on -# outpost-worker to take effect. Setting it only on a bot service does nothing. -# Staging MUST have SHADOW_MODE=true on the worker so it never posts to real -# communities. Production runs SHADOW_MODE=false. +# them to Discord/GitHub/Slack/Teams. TWO services read this flag and each gates a +# different point in the flow: +# - outpost-discord-bot gates at INGEST (thread-create.ts, message-create.ts +# divert to the silent shadow path) +# - outpost-worker gates at POST-BACK (the AI_RESPONSE handler checks it right +# before adapter.postResponse, and it also gates ONBOARDING_DIGEST) +# For Discord either gate alone stops a post. The worker's gate is the one that +# covers EVERY platform (GitHub, Slack, Teams) plus the digest job, so staging +# must have it set on outpost-worker — not only on a bot. +# Staging runs SHADOW_MODE=true; production runs false. SHADOW_MODE=false # ─── Discord Bot ───────────────────────────────────────────────────────────── diff --git a/.eslintrc.cjs b/.eslintrc.cjs new file mode 100644 index 00000000..966c86fd --- /dev/null +++ b/.eslintrc.cjs @@ -0,0 +1,24 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + root: true, + env: { + node: true, + es2022: true, + }, + parser: '@typescript-eslint/parser', + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + }, + plugins: ['@typescript-eslint'], + extends: [ + 'eslint:recommended', + 'plugin:@typescript-eslint/recommended', + ], + rules: { + '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/consistent-type-imports': 'warn', + }, + ignorePatterns: ['node_modules/', 'dist/', '.next/', '*.js', '!.eslintrc.js'], +}; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79c32001..cc75fce4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,11 +56,14 @@ jobs: - name: Build run: pnpm build - # Config lives in eslint.config.cjs (ESLint 9 flat config), so this needs no - # environment coaxing and behaves the same here, in a package directory, and - # in an editor's ESLint integration. - - name: Lint - run: pnpm lint + # Lint is NOT run here yet, even though this job is named "Lint, Typecheck & + # Test" and branch protection requires that check. Enabling it needs the + # ESLint 9 flat-config migration (this repo still uses .eslintrc.cjs, and + # ESLint 9 will not read it without ESLINT_USE_FLAT_CONFIG=false, which does + # not survive turbo's environment sanitization). That migration also changes + # lint resolution for editors, per-package invocations, and apps/web's + # separate Next config, so it is being landed as its own PR where those + # effects can be verified in isolation rather than bundled here. - name: Typecheck run: pnpm typecheck diff --git a/apps/worker/package.json b/apps/worker/package.json index 8d30fb15..ae6eb1a5 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -9,7 +9,7 @@ "dev": "tsx watch src/index.ts", "start": "node dist/index.js", "typecheck": "tsc --noEmit", - "lint": "eslint src/", + "lint": "echo 'no eslint config for worker yet'", "test": "vitest run" }, "dependencies": { diff --git a/docs/deployment.md b/docs/deployment.md index 285e05d9..ef1609cc 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -62,7 +62,12 @@ running one service at a time, and running several together needs a per-process One further asymmetry: the worker resolves `PORT ?? HEALTH_PORT ?? 3003` (`apps/worker/src/index.ts`), so a platform-injected `PORT` **overrides** `HEALTH_PORT` — and since its Dockerfile probes 3005 unconditionally, an injected `PORT` moves the listener while -the health check keeps checking 3005. The Teams bot reads only `HEALTH_PORT`. +the health check keeps checking 3005. + +Several services use `PORT` and `HEALTH_PORT` for **different** listeners rather than as +alternatives — the Teams bot serves health on `HEALTH_PORT` (3003) and the Bot Framework +endpoint on `PORT` (3978), and Linear sync reads both as well. Only the worker treats them as +a fallback chain. ## Environment Variables diff --git a/eslint.config.cjs b/eslint.config.cjs deleted file mode 100644 index db630fea..00000000 --- a/eslint.config.cjs +++ /dev/null @@ -1,70 +0,0 @@ -/** - * ESLint flat config for every workspace except `apps/web`. - * - * ESLint 9 looks for this file by default. Before it existed, linting depended on - * ESLINT_USE_FLAT_CONFIG=false to opt back into `.eslintrc.cjs` — which only worked where - * that variable happened to be set, so `pnpm lint` inside a package, an editor's ESLint - * integration, and any non-POSIX shell all failed. Defining the config here removes the - * variable from all of those paths. - * - * Translated from the former root `.eslintrc.cjs` via FlatCompat: same parser, same two - * extends, same three rule overrides. The ignore list is NOT a literal copy — see below. - * - * `apps/web` is deliberately excluded. It keeps its own `.eslintrc.cjs` (Next.js rules, - * including react-hooks) and is linted by `next lint` through its own package script. If - * this config applied there, it would shadow that eslintrc and ESLint would fail with - * "Definition for rule 'react-hooks/exhaustive-deps' was not found", since the Next and - * react-hooks plugins are not loaded here. Migrating web is still outstanding: `next lint` - * is removed in Next 16, and until then eslintrc lives on in that one package. - */ - -const { FlatCompat } = require('@eslint/eslintrc'); -const js = require('@eslint/js'); - -const compat = new FlatCompat({ - baseDirectory: __dirname, - recommendedConfig: js.configs.recommended, -}); - -module.exports = [ - { - // The former eslintrc ignored: node_modules/, dist/, .next/, *.js. - // Flat config needs `**/` to match at any depth — a bare `dist/` matches only the - // repo root. Three entries are additions, not translations: - // - apps/web/** — owned by next lint + apps/web/.eslintrc.cjs (see above) - // - **/generated/** — Prisma client output, not hand-written source - // - **/*.cjs, **/*.mjs — config files (this one, postcss.config.cjs); the old - // config ignored only `*.js`, so these were nominally in - // scope. Excluding them is a deliberate scope reduction. - ignores: [ - '**/node_modules/**', - '**/dist/**', - '**/.next/**', - '**/generated/**', - '**/*.js', - '**/*.cjs', - '**/*.mjs', - 'apps/web/**', - ], - }, - ...compat.config({ - // NOTE: no `root: true` here — that key is eslintrc-only and FlatCompat drops it - // silently. Flat config has no cascade, so there is nothing to root. - env: { - node: true, - es2022: true, - }, - parser: '@typescript-eslint/parser', - parserOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - }, - plugins: ['@typescript-eslint'], - extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended'], - rules: { - '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_' }], - '@typescript-eslint/no-explicit-any': 'error', - '@typescript-eslint/consistent-type-imports': 'warn', - }, - }), -]; diff --git a/package.json b/package.json index e72e7209..fda7ebc6 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "lint": "turbo run lint && eslint scripts/ vitest.config.ts", + "lint": "turbo run lint", "test": "turbo run test", "typecheck": "turbo run typecheck", "db:generate": "turbo run db:generate --filter=@copilotkit/outpost", @@ -16,8 +16,6 @@ "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\"" }, "devDependencies": { - "@eslint/eslintrc": "^3.3.5", - "@eslint/js": "^9.39.4", "@linear/sdk": "^81.0.0", "@typescript-eslint/eslint-plugin": "^8.58.2", "@typescript-eslint/parser": "^8.58.2", @@ -32,5 +30,6 @@ "engines": { "node": ">=20.0.0" }, - "dependencies": {} + "dependencies": { + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 63420a32..b3a28953 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,12 +8,6 @@ importers: .: devDependencies: - '@eslint/eslintrc': - specifier: ^3.3.5 - version: 3.3.5 - '@eslint/js': - specifier: ^9.39.4 - version: 9.39.4 '@linear/sdk': specifier: ^81.0.0 version: 81.0.0(graphql@16.13.2) diff --git a/turbo.json b/turbo.json index d638c6c5..307c058c 100644 --- a/turbo.json +++ b/turbo.json @@ -5,7 +5,7 @@ // pass from before the rule change. Deliberately narrow — listing a file no task // consumes (.prettierrc, the root vitest.config.ts) would invalidate every // package's cache on edits that cannot change any task's result. - "globalDependencies": ["eslint.config.cjs", "tsconfig.json"], + "globalDependencies": [".eslintrc.cjs", "tsconfig.json"], "tasks": { "build": { "dependsOn": ["^build"], From 72f63d3a34cc99daee1274dd6eac242af62b7458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:55:59 -0400 Subject: [PATCH 32/83] fix(web): pass NEXT_PUBLIC_* through to the client bundle build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NEXT_PUBLIC_* values are inlined into the browser bundle by `next build` rather than read at runtime, so they must be present during the build. Two layers were dropping them, and both had to be fixed for the value to land: 1. turbo runs envMode: strict, and the build task declared no env, so `turbo run build` handed next build an environment with no NEXT_PUBLIC_* at all (verified via --dry=json: configured [], inferred []). Declared env: ["NEXT_PUBLIC_*", "NODE_ENV"] on the build task. 2. apps/web/Dockerfile declared no ARG, so even a Railway-supplied build variable never entered the builder stage. Added ARG + matching ENV for NEXT_PUBLIC_AUTH_PROVIDER and NEXT_PUBLIC_SENTRY_DSN above the build step — ARG alone is insufficient because Next reads process.env, and placement below the RUN would have no effect. The production image builds with `pnpm turbo run build`, so today every NEXT_PUBLIC_* is baked in as `undefined`. It is currently masked: login reads NEXT_PUBLIC_AUTH_PROVIDER ?? 'credentials' and credentials is what is deployed, so client and server agree by accident. Setting AUTH_PROVIDER to github or oidc would have switched the server while the bundle kept rendering the credentials flow. Verified: turbo now reports specified.env as the two entries and lists NEXT_PUBLIC_AUTH_PROVIDER under configured when set, and the task hash changes with the value (cc083bf3 -> 96300fcc), so a changed value rebuilds rather than reusing a stale bundle. The docker build-arg path is NOT verified here — the daemon is not running in this environment; see the PR body for the one-command check. Note: NEXT_PUBLIC_SENTRY_DSN stays inert until the Sentry SDK is installed — apps/web/src/lib/sentry.ts is currently a no-op stub. --- apps/web/Dockerfile | 12 ++++++++++++ turbo.json | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 16fb0231..fba324aa 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -24,6 +24,18 @@ RUN pnpm install --frozen-lockfile COPY --from=pruner /app/out/full/ . # turbo prune omits the root tsconfig.json that packages/outpost/tsconfig.base.json extends COPY --from=pruner /app/tsconfig.json ./tsconfig.json + +# NEXT_PUBLIC_* values are inlined into the client bundle at build time, not read at +# runtime — so they must exist during `next build` or Next bakes in `undefined`, and +# setting them on the running service afterwards has no effect. Railway supplies service +# variables to Dockerfile builds as build args, so each one needs an ARG here; the ARG +# alone is not enough because Next reads process.env, hence the matching ENV. Both must +# appear ABOVE the build step to have any effect. +ARG NEXT_PUBLIC_AUTH_PROVIDER=credentials +ARG NEXT_PUBLIC_SENTRY_DSN= +ENV NEXT_PUBLIC_AUTH_PROVIDER=$NEXT_PUBLIC_AUTH_PROVIDER +ENV NEXT_PUBLIC_SENTRY_DSN=$NEXT_PUBLIC_SENTRY_DSN + RUN pnpm turbo run build --filter=@copilotkit/outpost-web # ── Stage 3: production image ──────────────────────────────────────────────── diff --git a/turbo.json b/turbo.json index aa60adf1..dd5c56d0 100644 --- a/turbo.json +++ b/turbo.json @@ -3,7 +3,14 @@ "tasks": { "build": { "dependsOn": ["^build"], - "outputs": ["dist/**", "**/dist/**", ".next/**"] + "outputs": ["dist/**", "**/dist/**", ".next/**"], + // turbo runs in strict env mode, so a task sees only the variables declared + // here. NEXT_PUBLIC_* are inlined into the client bundle by `next build`, so + // if they are missing at build time Next bakes in `undefined` and no runtime + // configuration can recover them. Declaring them also puts them in the cache + // key, so changing a value rebuilds instead of reusing a bundle built with + // the old one. + "env": ["NEXT_PUBLIC_*", "NODE_ENV"] }, "dev": { "cache": false, From 365c71bc56a6adfe23e0ee1970f35aea681153cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:08:16 -0400 Subject: [PATCH 33/83] fix(web): close CR findings on the NEXT_PUBLIC build-env fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 7-agent CR pass found four defects in the original 21-line change; five of the seven slots converged on the same items. Empty string was treated as a configured value. ARG NEXT_PUBLIC_SENTRY_DSN= gave the variable an empty default, so Next inlined '' rather than leaving it absent, and sentry.ts's 'SENTRY_DSN ?? NEXT_PUBLIC_SENTRY_DSN' accepts '' because it is not nullish. Dropped that ARG/ENV pair entirely — the consumer is still a no-op stub with no @sentry package, so wiring it bought nothing and cost a fallback. turbo's wildcard still covers it for when the SDK lands. The same hazard applied to the auth provider, with a worse outcome: a blank Railway variable would inline '', and login/page.tsx used '??', so AUTH_PROVIDER became '' and the page rendered a card with no sign-in control. Switched that consumer to '||' so empty means not-configured, and removed the ARG's default so the application's own default is the single source of truth rather than being repeated in the Dockerfile. Documented that the client's build-time value must agree with the server's runtime AUTH_PROVIDER — a mismatch does not fail loudly, it locks users out behind a misleading 'Invalid email or password'. docker-compose.yml built this Dockerfile while passing NEXT_PUBLIC_AUTH_PROVIDER only under 'environment:', which never reaches the bundle — reproducing the exact bug being fixed, masked only because the old ARG default happened to match. Added build.args. Moved 'env' off the shared build task onto '@copilotkit/outpost-web#build'. On the root task it hashed web-only variables into every package's cache key, so changing one busted unrelated caches. Pinned envMode: strict rather than relying on turbo 2's default, so the comments' premise cannot be invalidated by a future default change. Verified: web#build reports specified ['NEXT_PUBLIC_*','NODE_ENV'] and configured NEXT_PUBLIC_AUTH_PROVIDER, while @copilotkit/outpost#build reports both empty — the scoping works. Compose resolves the build arg. typecheck 10/10. A docker build with --build-arg NEXT_PUBLIC_AUTH_PROVIDER=github inlined the value (variable name absent from .next, literal present); the image is being rebuilt after these changes to re-confirm. --- apps/web/Dockerfile | 12 +++++++++--- apps/web/src/app/login/page.tsx | 10 +++++++++- docker-compose.yml | 5 +++++ turbo.json | 23 +++++++++++++++++------ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index fba324aa..04be8cdd 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -31,10 +31,16 @@ COPY --from=pruner /app/tsconfig.json ./tsconfig.json # variables to Dockerfile builds as build args, so each one needs an ARG here; the ARG # alone is not enough because Next reads process.env, hence the matching ENV. Both must # appear ABOVE the build step to have any effect. -ARG NEXT_PUBLIC_AUTH_PROVIDER=credentials -ARG NEXT_PUBLIC_SENTRY_DSN= +# +# ADDING A NEW NEXT_PUBLIC_* VARIABLE REQUIRES ITS OWN ARG + ENV PAIR HERE. turbo.json's +# wildcard covers turbo's side, but Docker has no wildcard for build args — a variable +# without a pair below is silently baked in as `undefined`. +# +# No default value is given deliberately. An unset build arg makes ENV an empty string, +# and consumers treat empty as absent (`||`, not `??`) so the application's own default +# applies. Repeating the default here instead would put it in two places that can drift. +ARG NEXT_PUBLIC_AUTH_PROVIDER ENV NEXT_PUBLIC_AUTH_PROVIDER=$NEXT_PUBLIC_AUTH_PROVIDER -ENV NEXT_PUBLIC_SENTRY_DSN=$NEXT_PUBLIC_SENTRY_DSN RUN pnpm turbo run build --filter=@copilotkit/outpost-web diff --git a/apps/web/src/app/login/page.tsx b/apps/web/src/app/login/page.tsx index 2a26dd3d..2264e65a 100644 --- a/apps/web/src/app/login/page.tsx +++ b/apps/web/src/app/login/page.tsx @@ -5,7 +5,15 @@ import { useSearchParams } from 'next/navigation'; import { Mountain, Github, KeyRound, Shield } from 'lucide-react'; import { Suspense, useState, useEffect } from 'react'; -const AUTH_PROVIDER = process.env.NEXT_PUBLIC_AUTH_PROVIDER ?? 'credentials'; +// `||` rather than `??` on purpose: this value is inlined at build time, and an unset or +// blank build arg inlines an empty string, which `??` would accept — leaving AUTH_PROVIDER +// as '' and rendering a login card with no sign-in control at all. Empty means "not +// configured", so it must fall through to the default. +// +// This must agree with the server's AUTH_PROVIDER (see src/lib/auth.ts). The two are read +// from different places — this one at build time, the server's at runtime — so a mismatch +// does not fail loudly: it locks users out behind a misleading "Invalid email or password". +const AUTH_PROVIDER = process.env.NEXT_PUBLIC_AUTH_PROVIDER || 'credentials'; function CredentialsForm() { const [email, setEmail] = useState(''); diff --git a/docker-compose.yml b/docker-compose.yml index 295497f8..5445e33c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,11 @@ services: build: context: . dockerfile: apps/web/Dockerfile + # NEXT_PUBLIC_* must be passed as a BUILD arg, not runtime environment — Next + # inlines it into the client bundle during `next build`, so the `environment:` + # entry below has no effect on what the browser receives. + args: + NEXT_PUBLIC_AUTH_PROVIDER: ${NEXT_PUBLIC_AUTH_PROVIDER:-credentials} container_name: outpost-web ports: - '3000:3000' diff --git a/turbo.json b/turbo.json index dd5c56d0..524c9700 100644 --- a/turbo.json +++ b/turbo.json @@ -1,15 +1,26 @@ { "$schema": "https://turbo.build/schema.json", + // Pinned rather than relying on turbo 2's default, so the reasoning below cannot be + // invalidated by a future change to that default: in strict mode a task sees only the + // variables it declares. + "envMode": "strict", "tasks": { "build": { + "dependsOn": ["^build"], + "outputs": ["dist/**", "**/dist/**", ".next/**"] + }, + // Scoped to the web package rather than declared on the shared `build` task. + // NEXT_PUBLIC_* only affects the Next app, and declaring it on the root task would + // hash web-only variables into every package's cache key, busting unrelated caches + // whenever one changed. + // + // These are inlined into the client bundle by `next build`, so if they are absent + // at build time Next bakes in `undefined` and no runtime configuration can recover + // them. Declaring them here also puts them in the cache key, so a changed value + // rebuilds instead of reusing a bundle built with the old one. + "@copilotkit/outpost-web#build": { "dependsOn": ["^build"], "outputs": ["dist/**", "**/dist/**", ".next/**"], - // turbo runs in strict env mode, so a task sees only the variables declared - // here. NEXT_PUBLIC_* are inlined into the client bundle by `next build`, so - // if they are missing at build time Next bakes in `undefined` and no runtime - // configuration can recover them. Declaring them also puts them in the cache - // key, so changing a value rebuilds instead of reusing a bundle built with - // the old one. "env": ["NEXT_PUBLIC_*", "NODE_ENV"] }, "dev": { From 8f0d73a8e38cd8e0297b897dfd4057b68bb4c836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:44:31 -0400 Subject: [PATCH 34/83] fix(ai): stop the bot confirming unverified bugs and hedging in public Three fixes to what the support bot says on public surfaces, all prompted by CopilotKit/CopilotKit#6167 where the bot posted "Bug Confirmed" for a cursor-jump report it never reproduced, invented CSS class names to explain it, and told maintainers what to fix. 1. Grounding rules in the generator's system prompt. The generator is a single stateless Claude call over Pathfinder docs results - no repo access, no repro, no tests - so the prompt now forbids confirming bugs, asserting root causes, naming identifiers absent from the documentation context, restating a hedged hypothesis as fact, and prescribing fixes to CopilotKit internals. Escalating beats a plausible-sounding guess. 2. The disclaimer no longer hedges about the response itself. "and may be incomplete" is gone from every variant; all copy now opens with "This is an AI-generated response." Confidence is expressed by escalating, not by telling the reader to distrust an answer we chose to post. Copy lives in three exported constants in formatter.ts so no surface can drift. 3. No more "Ticket TKT-xxx created" comment on GitHub issues and discussions. The ticket id is internal and the AI response lands in the same thread moments later. Discord/Slack/Teams acknowledgments are untouched - those are chat surfaces where an ack reads as the bot picking the thread up, not as a permanent comment on a public issue. Tests: grounding-rule coverage in generator.test.ts, no-hedge assertions across every confidence band in pipeline.test.ts and every platform in formatter.test.ts, and the two GitHub webhook tests now assert no system message is posted. --- .../src/__tests__/discussion-created.test.ts | 11 +--- .../src/__tests__/issues-opened.test.ts | 12 ++--- .../src/webhooks/discussion-created.ts | 15 +----- apps/github-app/src/webhooks/issues-opened.ts | 14 ++---- packages/outpost/ai/src/formatter.test.ts | 36 ++++++++++++- packages/outpost/ai/src/formatter.ts | 20 +++++++- packages/outpost/ai/src/generator.test.ts | 50 ++++++++++++++++++- packages/outpost/ai/src/generator.ts | 25 +++++++++- packages/outpost/ai/src/index.ts | 9 +++- packages/outpost/ai/src/pipeline.test.ts | 14 +++++- packages/outpost/ai/src/pipeline.ts | 15 ++++-- 11 files changed, 166 insertions(+), 55 deletions(-) diff --git a/apps/github-app/src/__tests__/discussion-created.test.ts b/apps/github-app/src/__tests__/discussion-created.test.ts index 58f1ab0e..81f21118 100644 --- a/apps/github-app/src/__tests__/discussion-created.test.ts +++ b/apps/github-app/src/__tests__/discussion-created.test.ts @@ -141,18 +141,11 @@ describe('handleDiscussionCreated', () => { }); }); - it('posts acknowledgment via adapter.postSystemMessage with discussion node_id', async () => { + it('does not post a ticket-created acknowledgment comment on the discussion', async () => { const event = makeEvent(); await handleDiscussionCreated(event); - expect(mockPostSystemMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'ticket-disc-id', - discussionNodeId: 'D_kwDOTest1234', - source: 'GITHUB_DISCUSSION', - }), - expect.stringContaining('TKT-DS01'), - ); + expect(mockPostSystemMessage).not.toHaveBeenCalled(); }); it('handles discussions with no body gracefully', async () => { diff --git a/apps/github-app/src/__tests__/issues-opened.test.ts b/apps/github-app/src/__tests__/issues-opened.test.ts index fa658989..4506ee3b 100644 --- a/apps/github-app/src/__tests__/issues-opened.test.ts +++ b/apps/github-app/src/__tests__/issues-opened.test.ts @@ -141,17 +141,13 @@ describe('handleIssueOpened', () => { }); }); - it('posts an acknowledgment via adapter.postSystemMessage', async () => { + it('does not post a ticket-created acknowledgment comment on the issue', async () => { const event = makeEvent(); await handleIssueOpened(event); - expect(mockPostSystemMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: 'ticket-internal-id', - source: 'GITHUB_ISSUE', - }), - expect.stringContaining('TKT-GH01'), - ); + // The internal ticket id is noise on a public issue — the AI response is + // the bot's only comment in the thread. + expect(mockPostSystemMessage).not.toHaveBeenCalled(); }); it('handles parse failure gracefully', async () => { diff --git a/apps/github-app/src/webhooks/discussion-created.ts b/apps/github-app/src/webhooks/discussion-created.ts index 77e015ff..ca802aaa 100644 --- a/apps/github-app/src/webhooks/discussion-created.ts +++ b/apps/github-app/src/webhooks/discussion-created.ts @@ -57,19 +57,8 @@ export async function handleDiscussionCreated( }, }); - // Post acknowledgment comment on the discussion - // Store the discussion node_id on the ticket ref for routing - const ticketRef = { - id: result.ticketId, - sourceId: `${repository.full_name}#${discussion.number}`, - channel: repository.full_name, - source: 'GITHUB_DISCUSSION' as const, - discussionNodeId: discussion.node_id, - }; - await adapter.postSystemMessage( - ticketRef as Parameters[0], - `\uD83C\uDFAB Ticket ${result.displayId} created. Our AI assistant is reviewing your question...`, - ); + // Intentionally no "Ticket TKT-\u2026 created" acknowledgment comment \u2014 see + // the matching note in issues-opened.ts. console.log( `[GitHub App] Created ticket ${result.displayId} for discussion "${discussion.title}"`, diff --git a/apps/github-app/src/webhooks/issues-opened.ts b/apps/github-app/src/webhooks/issues-opened.ts index 2389e36b..b00b4da8 100644 --- a/apps/github-app/src/webhooks/issues-opened.ts +++ b/apps/github-app/src/webhooks/issues-opened.ts @@ -57,17 +57,9 @@ export async function handleIssueOpened( }, }); - // Post acknowledgment comment on the issue - const ticketRef = { - id: result.ticketId, - sourceId: `${repository.full_name}#${issue.number}`, - channel: repository.full_name, - source: 'GITHUB_ISSUE' as const, - }; - await adapter.postSystemMessage( - ticketRef as Parameters[0], - `\uD83C\uDFAB Ticket ${result.displayId} created. Our AI assistant is reviewing your issue...`, - ); + // Intentionally no "Ticket TKT-\u2026 created" acknowledgment comment. The + // ticket id is internal, and the AI response lands in the same thread + // moments later \u2014 the ack was pure noise on a public issue. console.log( `[GitHub App] Created ticket ${result.displayId} for issue ${repository.full_name}#${issue.number}`, diff --git a/packages/outpost/ai/src/formatter.test.ts b/packages/outpost/ai/src/formatter.test.ts index bdc83987..7cc8feef 100644 --- a/packages/outpost/ai/src/formatter.test.ts +++ b/packages/outpost/ai/src/formatter.test.ts @@ -1,5 +1,29 @@ import { describe, it, expect } from 'vitest'; -import { ResponseFormatter } from './formatter.js'; +import { + AI_DISCLAIMER, + AI_DISCLAIMER_ESCALATED, + AI_DISCLAIMER_REVIEWED, + ResponseFormatter, +} from './formatter.js'; + +describe('disclaimer copy', () => { + it('opens every variant with the plain AI-generated sentence', () => { + expect(AI_DISCLAIMER).toBe('This is an AI-generated response.'); + expect(AI_DISCLAIMER_ESCALATED.startsWith(AI_DISCLAIMER)).toBe(true); + expect(AI_DISCLAIMER_REVIEWED.startsWith(AI_DISCLAIMER)).toBe(true); + }); + + it('never volunteers a judgement about the response quality', () => { + for (const text of [AI_DISCLAIMER, AI_DISCLAIMER_ESCALATED, AI_DISCLAIMER_REVIEWED]) { + expect(text).not.toMatch(/incomplete|inaccurate|may not be|unreliable/i); + } + }); + + it('only the escalated variant promises a follow-up in-thread', () => { + expect(AI_DISCLAIMER_ESCALATED).toContain("We've escalated this to our engineering team"); + expect(AI_DISCLAIMER_REVIEWED).not.toContain('escalated'); + }); +}); describe('ResponseFormatter', () => { const formatter = new ResponseFormatter(); @@ -58,6 +82,16 @@ describe('ResponseFormatter', () => { expect(result.text).toContain('AI-generated response'); }); + it('never hedges about completeness on any platform', () => { + for (const platform of ['discord', 'github', 'slack', 'teams', 'web'] as const) { + const result = formatter.format('Answer text', platform, { + addDisclaimer: true, + }); + expect(result.text).not.toMatch(/may be incomplete|might be incomplete/i); + expect(result.text).toContain(AI_DISCLAIMER); + } + }); + it('should use custom disclaimer text', () => { const result = formatter.format('Answer text', 'discord', { addDisclaimer: true, diff --git a/packages/outpost/ai/src/formatter.ts b/packages/outpost/ai/src/formatter.ts index fd879b5a..48845e2d 100644 --- a/packages/outpost/ai/src/formatter.ts +++ b/packages/outpost/ai/src/formatter.ts @@ -9,6 +9,24 @@ const GITHUB_FOOTER = '\n\n---\n🤖 Generated by CopilotKit AI Support · const WEB_FOOTER = '\n\n---\n*Powered by CopilotKit AI*'; +/** + * The disclaimer sentence every externally-visible AI response opens with. + * + * Deliberately narrow: it states WHAT wrote the response, never volunteers a + * judgement about the response's quality. Copy like "and may be incomplete" + * must never ship to a public GitHub issue or Discord thread — it invites the + * reader to distrust an answer we chose to post. Confidence is handled by the + * pipeline (a low-confidence answer escalates or isn't sent), not by hedging in + * front of the user. + */ +export const AI_DISCLAIMER = 'This is an AI-generated response.'; + +/** Disclaimer for responses the worker will actually enqueue an escalation for. */ +export const AI_DISCLAIMER_ESCALATED = `${AI_DISCLAIMER} We've escalated this to our engineering team — someone will follow up in this thread shortly.`; + +/** Disclaimer for responses that post without a guaranteed human follow-up. */ +export const AI_DISCLAIMER_REVIEWED = `${AI_DISCLAIMER} A member of our team will review and follow up if needed.`; + /** * Formats AI-generated responses for different platform targets. * @@ -26,7 +44,7 @@ export class ResponseFormatter { options?: { addDisclaimer?: boolean; disclaimerText?: string }, ): FormattedResponse { const disclaimer = options?.addDisclaimer - ? `> ⚠️ ${options.disclaimerText ?? 'This is an AI-generated response. A member of our team will review it shortly.'}\n\n` + ? `> ⚠️ ${options.disclaimerText ?? AI_DISCLAIMER_REVIEWED}\n\n` : ''; const fullText = disclaimer + text; diff --git a/packages/outpost/ai/src/generator.test.ts b/packages/outpost/ai/src/generator.test.ts index 501ad047..59e5bec7 100644 --- a/packages/outpost/ai/src/generator.test.ts +++ b/packages/outpost/ai/src/generator.test.ts @@ -1,6 +1,11 @@ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import { LLMock } from '@copilotkit/aimock'; -import { ResponseGenerator, buildChannelGuidance } from './generator.js'; +import { + GROUNDING_RULES, + SYSTEM_PROMPT_PREFIX, + ResponseGenerator, + buildChannelGuidance, +} from './generator.js'; import { ConfidenceLevel } from './types.js'; import type { SearchResult } from './types.js'; @@ -193,6 +198,49 @@ describe('ResponseGenerator', () => { }); }); + // Regression for CopilotKit/CopilotKit#6167: the bot posted "Bug Confirmed" + // with invented CSS class names for a report it never reproduced. The + // generator is a single stateless call over docs search — it has no repo + // access and runs no tests — so the prompt has to forbid those claims. + describe('GROUNDING_RULES', () => { + it('states the model has not read source, reproduced, or tested', () => { + expect(GROUNDING_RULES).toContain('have NOT read'); + expect(GROUNDING_RULES).toContain('reproduced'); + expect(GROUNDING_RULES).toContain('run any test'); + }); + + it('forbids confirming a bug or asserting a root cause', () => { + expect(GROUNDING_RULES).toContain('Never confirm a bug'); + expect(GROUNDING_RULES).toContain('bug confirmed'); + expect(GROUNDING_RULES).toContain('root cause is'); + }); + + it('restricts identifiers to ones present in the documentation context', () => { + expect(GROUNDING_RULES).toContain('appear verbatim in the Documentation Context'); + expect(GROUNDING_RULES).toContain('CSS class names'); + }); + + it('requires causal claims to stay marked as hypotheses', () => { + expect(GROUNDING_RULES).toContain('hypothesis'); + expect(GROUNDING_RULES).toContain('never restate it as established fact'); + }); + + it('forbids prescribing fixes to CopilotKit internals', () => { + expect(GROUNDING_RULES).toContain('Do not prescribe fixes to CopilotKit'); + }); + + it('prefers escalation over a plausible-sounding guess', () => { + expect(GROUNDING_RULES).toContain('escalating to the team'); + }); + + it('is wired into the system prompt and overrides the personality rules', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain(GROUNDING_RULES); + expect(GROUNDING_RULES).toContain( + 'override the personality and formatting rules above', + ); + }); + }); + describe('generateStream', () => { it('should yield text chunks from streaming response', async () => { mock.onMessage(/./, { diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index a51ac15c..beb9333b 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -5,7 +5,26 @@ import type { GeneratedResponse, PipelineContext, SearchResult, TokenUsage } fro import { ConfidenceLevel, classifyConfidence } from './types.js'; import { config } from './config.js'; -const SYSTEM_PROMPT_PREFIX = `You are an AI support assistant for CopilotKit, an open-source framework for building AI copilots, chatbots, and AI-powered UIs. +/** + * Epistemic guardrails. The generator is a SINGLE stateless model call over + * documentation search results — it cannot read CopilotKit's source, cannot run + * a repro, and cannot execute tests. Without these rules it will happily assert + * a confirmed root cause built from generic framework priors (see + * CopilotKit/CopilotKit#6167, where the bot posted "Bug Confirmed" plus invented + * CSS class names for a cursor-jump report it never reproduced). + * + * Every rule here exists to keep the response's claims inside what the provided + * Documentation Context actually supports. + */ +export const GROUNDING_RULES = `Grounding rules (these override the personality and formatting rules above when they conflict): +- You have NOT read CopilotKit's source code, reproduced the user's problem, or run any test. Never write or imply otherwise. +- Never confirm a bug. Do not write "bug confirmed", "this is a real bug", "known issue", "root cause is", or "the fix is" about behavior you cannot see. Acknowledge the report and say engineering will verify. +- Only name identifiers — file paths, CSS class names, component names, props, hooks, config keys, version numbers — that appear verbatim in the Documentation Context. If it is not there, describe the concept in prose instead of guessing a name. +- Mark any causal explanation as a hypothesis exactly once ("one possibility is…"), and never restate it as established fact later in the same response. If you hedge a claim, do not close by asserting it. +- Do not prescribe fixes to CopilotKit's internals or tell maintainers what to change; that call is theirs. Workarounds the user can apply in their own code are fine. +- Prefer "I don't have enough to answer this — escalating to the team" over a plausible-sounding answer assembled from general framework knowledge.`; + +export const SYSTEM_PROMPT_PREFIX = `You are an AI support assistant for CopilotKit, an open-source framework for building AI copilots, chatbots, and AI-powered UIs. Your personality: - Conversational and helpful, not robotic @@ -18,7 +37,9 @@ Formatting rules: - Use markdown formatting throughout - Wrap code in fenced code blocks with language tags - Use bold for emphasis on key concepts -- Keep paragraphs concise — prefer bullets over walls of text`; +- Keep paragraphs concise — prefer bullets over walls of text + +${GROUNDING_RULES}`; /** * Per-channel guidance so the response never redirects the user to the channel diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 1aef972d..8b777586 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -1,9 +1,14 @@ export { PathfinderClient } from './pathfinder.js'; -export { ResponseGenerator } from './generator.js'; +export { ResponseGenerator, GROUNDING_RULES, SYSTEM_PROMPT_PREFIX } from './generator.js'; export { ConfidenceScorer } from './confidence.js'; export type { ConfidenceAssessment } from './confidence.js'; export { TicketClassifier } from './classifier.js'; -export { ResponseFormatter } from './formatter.js'; +export { + AI_DISCLAIMER, + AI_DISCLAIMER_ESCALATED, + AI_DISCLAIMER_REVIEWED, + ResponseFormatter, +} from './formatter.js'; export { AIPipeline } from './pipeline.js'; export { analyzeSentiment } from './sentiment.js'; export { scoreEngagement } from './engagement.js'; diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 5c4e52fd..bf8067e8 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -193,17 +193,27 @@ describe('AIPipeline', () => { const text = await disclaimerFor(0.45); // Was the bug: 0.45 is LOW but never escalated, so no false promise. expect(text).not.toContain('escalated'); - expect(text).toContain('may be incomplete'); expect(text).toContain('will review and follow up'); }); it('uses the neutral MEDIUM copy for scores in [0.5, 0.8)', async () => { const text = await disclaimerFor(0.6); expect(text).not.toContain('escalated'); - expect(text).not.toContain('may be incomplete'); expect(text).toContain('A member of our team will review'); }); + // No externally-visible disclaimer may hedge about the response's own + // completeness — that copy invites the reader to distrust an answer we + // chose to post. Confidence is expressed by escalating, not by hedging. + it.each([0.1, 0.3, 0.45, 0.6, 0.95])( + 'never hedges about completeness at score %s', + async (score) => { + const text = await disclaimerFor(score); + expect(text).not.toMatch(/may be incomplete|might be incomplete|may not be accurate/i); + expect(text).toContain('This is an AI-generated response.'); + }, + ); + it('should handle Pathfinder failure gracefully', async () => { mockSearchDocs.mockRejectedValueOnce(new Error('MCP down')); diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index 4360b167..58cf5c03 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -12,7 +12,11 @@ import { PathfinderClient } from './pathfinder.js'; import { ResponseGenerator } from './generator.js'; import { ConfidenceScorer } from './confidence.js'; import { TicketClassifier } from './classifier.js'; -import { ResponseFormatter } from './formatter.js'; +import { + AI_DISCLAIMER_ESCALATED, + AI_DISCLAIMER_REVIEWED, + ResponseFormatter, +} from './formatter.js'; import { validateConfig } from './config.js'; /** @@ -149,13 +153,14 @@ export class AIPipeline { // MEDIUM_THRESHOLD). Otherwise a score in [ESCALATE, MEDIUM_THRESHOLD) // is LOW but never escalated, so the reporter is promised a follow-up // that never comes. + // + // Neither variant may hedge about the response's completeness — see the + // AI_DISCLAIMER doc comment in formatter.ts. const needsDisclaimer = finalConfidence !== ConfidenceLevel.HIGH; const willEscalate = finalConfidenceScore < AI_CONFIDENCE.ESCALATE; const disclaimerText = willEscalate - ? "This is an AI-generated response and may be incomplete. We've escalated this to our engineering team — someone will follow up in this thread shortly." - : finalConfidence === ConfidenceLevel.LOW - ? 'This is an AI-generated response and may be incomplete. A member of our team will review and follow up if needed.' - : 'This is an AI-generated response. A member of our team will review and follow up if needed.'; + ? AI_DISCLAIMER_ESCALATED + : AI_DISCLAIMER_REVIEWED; const formatted = this.formatter.format(generatedResponse.text, options.source, { addDisclaimer: needsDisclaimer, From 3d22e10f25b28e9b342a4268656af0ae1534e688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:04:14 -0400 Subject: [PATCH 35/83] fix(ai): score responses on groundedness and withhold ungrounded ones Confidence scoring measured retrieval quality and ignored whether the answer stayed inside what was retrieved. assessConfidence never read the response at all (the parameter was literally _response, unused), and the LLM scorer's rubric rewarded specificity - "cites specific features, APIs, or code patterns" - without asking whether the citations were real. A confident fabrication therefore scored exactly as high as a cited answer. New ai/groundedness.ts runs no model call: regexes over the response plus a substring lookup against the sources it was generated from. - Unverified claims ("bug confirmed", "root cause is", "the fix is", claims of having reproduced or tested) - any one suppresses the response - CopilotKit identifiers absent from every source - one penalizes, two suppress. Scoped to CSS classes and backticked tokens carrying our own name, so generic React vocabulary and @copilotkit package specifiers never trip it - Hedge density beyond a two-marker allowance - penalizes, never suppresses Wiring: - generator.assessConfidence now reads the response and deducts the penalty - pipeline applies the penalty AFTER feedback calibration, so aggregate thumbs-up can't buy back a fabrication, and clamps a suppressed response below the escalation gate - confidence.ts rubric weighs groundedness above specificity - ai-response.ts withholds a suppressed response from the platform and escalates That last one is the point. Confidence never gated the post-back - a low score only picked the disclaimer and queued an escalation - so lowering a number would not have kept a fabrication out of a public thread. A suppressed response now never posts, still persists as suggestedResponse for a human to edit, and escalates regardless of score. The clamp exists because a test found the penalty cap insufficient on its own: a top retrieval score plus maximum positive calibration minus the capped penalty lands on exactly 0.4, and the escalation gate is < 0.4, so it would have silently declined to escalate. Tests: 23 cases in groundedness.test.ts including the verbatim #6167 response as a fixture, 4 pipeline cases covering penalty/suppression/calibration ordering, and 3 handler cases proving no post, escalation regardless of score, and the draft surviving for a human. --- packages/outpost/ai/src/confidence.ts | 6 + packages/outpost/ai/src/generator.ts | 11 +- packages/outpost/ai/src/groundedness.test.ts | 214 ++++++++++++++++++ packages/outpost/ai/src/groundedness.ts | 183 +++++++++++++++ packages/outpost/ai/src/index.ts | 6 + packages/outpost/ai/src/pipeline.test.ts | 59 +++++ packages/outpost/ai/src/pipeline.ts | 36 +++ packages/outpost/ai/src/types.ts | 11 + .../queue/src/__tests__/ai-response.test.ts | 92 ++++++++ .../outpost/queue/src/handlers/ai-response.ts | 35 ++- 10 files changed, 644 insertions(+), 9 deletions(-) create mode 100644 packages/outpost/ai/src/groundedness.test.ts create mode 100644 packages/outpost/ai/src/groundedness.ts diff --git a/packages/outpost/ai/src/confidence.ts b/packages/outpost/ai/src/confidence.ts index 8dab276b..840faf6b 100644 --- a/packages/outpost/ai/src/confidence.ts +++ b/packages/outpost/ai/src/confidence.ts @@ -18,6 +18,12 @@ Evaluate these factors: 2. **Coverage**: Does the response address all parts of the question? 3. **Specificity**: Is the response specific and actionable, or vague and generic? 4. **Accuracy indicators**: Does the response cite specific features, APIs, or code patterns that exist in CopilotKit? +5. **Groundedness**: Is every specific claim traceable to the search results above? The assistant that wrote this response could not read CopilotKit's source, reproduce the user's problem, or run any test — it only had these search results. Score LOW when the response: + - confirms a bug, asserts a root cause, or claims to have reproduced or tested anything + - names a file, CSS class, component, prop, hook, or version that does not appear in the search results + - hedges ("likely", "may vary") and then states the same claim as fact + +Specificity that is not grounded is worse than a vague answer — a confident fabrication is the failure mode this score exists to catch. Weigh groundedness above specificity when the two conflict. Respond with ONLY a JSON object (no markdown, no explanation outside the JSON): { diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index beb9333b..ea7c518a 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -3,6 +3,7 @@ import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import type { PlatformTarget } from '@copilotkit/outpost/shared'; import type { GeneratedResponse, PipelineContext, SearchResult, TokenUsage } from './types.js'; import { ConfidenceLevel, classifyConfidence } from './types.js'; +import { assessGroundedness } from './groundedness.js'; import { config } from './config.js'; /** @@ -229,14 +230,20 @@ export class ResponseGenerator { return messages; } - private assessConfidence(sources: SearchResult[], _response: string): number { + private assessConfidence(sources: SearchResult[], response: string): number { if (sources.length === 0) return 0.2; const avgRelevance = this.avgScore(sources); const sourceCountBonus = Math.min(sources.length * 0.05, 0.15); // Base confidence on source quality + count - return Math.min(avgRelevance + sourceCountBonus, 1.0); + const retrievalScore = Math.min(avgRelevance + sourceCountBonus, 1.0); + + // Retrieval quality alone says nothing about whether the answer stayed + // inside those sources. Deduct for claims the response is not entitled + // to make, so a fabrication can't inherit a good docs match's score. + const { penalty } = assessGroundedness(response, sources); + return Math.max(0, retrievalScore - penalty); } private avgScore(sources: SearchResult[]): number { diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts new file mode 100644 index 00000000..1696d5fe --- /dev/null +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect } from 'vitest'; +import { + assessGroundedness, + extractCopilotKitIdentifiers, + MAX_GROUNDEDNESS_PENALTY, +} from './groundedness.js'; +import type { SearchResult } from './types.js'; + +const source = (content: string, title = 'CopilotChat'): SearchResult => ({ + title, + content, + score: 0.9, + sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat', +}); + +/** The docs page the #6167 query actually retrieved — no cursor/CSS internals on it. */ +const CHAT_DOCS = [ + source( + 'CopilotChat renders a chat window. Use the `CopilotChat` component with the ' + + '`instructions` prop. Slots let you replace the input via the `input` prop.', + ), +]; + +describe('extractCopilotKitIdentifiers', () => { + it('picks up CopilotKit CSS class selectors', () => { + const ids = extractCopilotKitIdentifiers( + 'Override `.copilotKitInputControls` and .copilotKitInputControlsExpanded to fix it.', + ); + expect(ids).toContain('copilotKitInputControls'); + expect(ids).toContain('copilotKitInputControlsExpanded'); + }); + + it('picks up backticked CopilotKit identifiers', () => { + expect(extractCopilotKitIdentifiers('Call `useCopilotChatInternals()` first.')).toEqual([]); + expect(extractCopilotKitIdentifiers('Call `useCopilotkitInternals` first.')).toContain( + 'useCopilotkitInternals', + ); + }); + + it('ignores generic React vocabulary so real answers are not penalized', () => { + const ids = extractCopilotKitIdentifiers( + 'Use `useRef`, `useLayoutEffect` and `setSelectionRange` to restore the cursor.', + ); + expect(ids).toEqual([]); + }); + + it('ignores @copilotkit package specifiers', () => { + const ids = extractCopilotKitIdentifiers( + 'Install `@copilotkit/react-core` and `@copilotkit/react-ui`.', + ); + expect(ids).toEqual([]); + }); + + it('ignores prose and code fences that are not identifiers', () => { + const ids = extractCopilotKitIdentifiers( + 'The `CopilotKit provider wraps your app` sentence is not an identifier.', + ); + expect(ids).toEqual([]); + }); + + it('deduplicates repeated mentions', () => { + const ids = extractCopilotKitIdentifiers( + '.copilotKitInput and .copilotKitInput again and `copilotKitInput`', + ); + expect(ids).toEqual(['copilotKitInput']); + }); +}); + +describe('assessGroundedness', () => { + it('gives a grounded answer no penalty and does not suppress it', () => { + const response = + 'You can replace the chat input with the `input` prop on the `CopilotChat` component. ' + + 'That keeps your own state, so you control the cursor.'; + + const result = assessGroundedness(response, CHAT_DOCS); + + expect(result.penalty).toBe(0); + expect(result.suppress).toBe(false); + expect(result.reasons).toEqual([]); + }); + + it('flags a confirmed-bug claim and suppresses the response', () => { + const result = assessGroundedness( + '## Bug Confirmed: Cursor Jump in Expanded Mode\n\nThanks for the repro steps!', + CHAT_DOCS, + ); + + expect(result.unverifiedClaims).toContain('"bug confirmed"'); + expect(result.penalty).toBeGreaterThan(0); + expect(result.suppress).toBe(true); + }); + + it.each([ + ['Root Cause: React resets the cursor.', 'asserts a root cause'], + ['This is a real bug worth fixing in the core.', 'asserts the report is a real bug'], + ['This is a known issue in the chat input.', 'claims a known bug'], + ['The fix is to save and restore the selection.', 'asserts the fix'], + ['I reproduced this locally on the latest version.', 'claims to have reproduced or tested'], + ['We ran the tests and they pass.', 'claims to have reproduced or tested'], + ])('suppresses %j', (response, expectedLabel) => { + const result = assessGroundedness(response, CHAT_DOCS); + expect(result.unverifiedClaims).toContain(expectedLabel); + expect(result.suppress).toBe(true); + }); + + it('penalizes an identifier absent from every source', () => { + const result = assessGroundedness( + 'Override `.copilotKitInputControls` to force compact mode.', + CHAT_DOCS, + ); + + expect(result.unsourcedIdentifiers).toEqual(['copilotKitInputControls']); + expect(result.penalty).toBeCloseTo(0.15, 5); + // A single identifier is a penalty, not a block — it could be a typo. + expect(result.suppress).toBe(false); + }); + + it('does not penalize an identifier the sources actually document', () => { + const result = assessGroundedness( + 'Style the input with `.copilotKitInput` as shown in the docs.', + [source('Override .copilotKitInput to restyle the chat input.')], + ); + + expect(result.unsourcedIdentifiers).toEqual([]); + expect(result.penalty).toBe(0); + }); + + it('matches identifiers against source titles as well as bodies', () => { + const result = assessGroundedness('Use `copilotKitSidebar` here.', [ + source('Layout options for the sidebar.', 'copilotKitSidebar reference'), + ]); + + expect(result.unsourcedIdentifiers).toEqual([]); + }); + + it('suppresses once two identifiers are invented', () => { + const result = assessGroundedness( + 'Override `.copilotKitInputControls` and `.copilotKitInputControlsExpanded`.', + CHAT_DOCS, + ); + + expect(result.unsourcedIdentifiers).toHaveLength(2); + expect(result.suppress).toBe(true); + }); + + it('allows a couple of hedges but penalizes a pile of them', () => { + const twoHedges = assessGroundedness( + 'This is likely a re-render, and the class names may vary by version.', + CHAT_DOCS, + ); + expect(twoHedges.penalty).toBe(0); + + const manyHedges = assessGroundedness( + 'This is likely a re-render. It probably resets state. The names may vary. ' + + 'I think the layout might be recalculating, though I am not sure.', + CHAT_DOCS, + ); + expect(manyHedges.penalty).toBeGreaterThan(0); + expect(manyHedges.hedgeCount).toBeGreaterThan(2); + // Hedging alone is a smell, not a fabrication — never blocks on its own. + expect(manyHedges.suppress).toBe(false); + }); + + it('caps the total penalty', () => { + const kitchenSink = + 'Bug Confirmed. Root cause is a re-render. This is a known issue. The fix is simple. ' + + 'I reproduced it. Override `.copilotKitA`, `.copilotKitB`, `.copilotKitC`, ' + + '`.copilotKitD`, `.copilotKitE`. Likely, probably, might be, may vary, I think.'; + + const result = assessGroundedness(kitchenSink, CHAT_DOCS); + + expect(result.penalty).toBe(MAX_GROUNDEDNESS_PENALTY); + expect(result.suppress).toBe(true); + }); + + it('treats an empty response and empty sources as ungraded rather than throwing', () => { + expect(assessGroundedness('', CHAT_DOCS).penalty).toBe(0); + expect(assessGroundedness('', CHAT_DOCS).suppress).toBe(false); + + // No sources means every identifier is unsourced — that is the correct read. + const noSources = assessGroundedness('Use `.copilotKitInput` here.', []); + expect(noSources.unsourcedIdentifiers).toEqual(['copilotKitInput']); + }); + + it('tolerates sources with missing title or content', () => { + const partial = [{ title: '', content: '', score: 0.5 } as SearchResult]; + expect(() => assessGroundedness('Anything at all.', partial)).not.toThrow(); + }); + + // The exact response from CopilotKit/CopilotKit#6167, condensed. + it('would have caught the #6167 response', () => { + const response = [ + '## Bug Confirmed: Cursor Jump in Expanded Mode', + 'This is a known React controlled input bug.', + '## Root Cause (Likely)', + 'In expanded mode, the input component probably re-renders on every keystroke.', + '```css', + '.copilotKitInputControls { flex-direction: row !important; }', + '.copilotKitInputControlsExpanded { display: none !important; }', + '```', + 'This is a real bug worth fixing in the core.', + ].join('\n'); + + const result = assessGroundedness(response, CHAT_DOCS); + + expect(result.suppress).toBe(true); + expect(result.unverifiedClaims.length).toBeGreaterThanOrEqual(2); + expect(result.unsourcedIdentifiers).toEqual([ + 'copilotKitInputControls', + 'copilotKitInputControlsExpanded', + ]); + expect(result.penalty).toBe(MAX_GROUNDEDNESS_PENALTY); + }); +}); diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts new file mode 100644 index 00000000..ca6008a7 --- /dev/null +++ b/packages/outpost/ai/src/groundedness.ts @@ -0,0 +1,183 @@ +import type { SearchResult } from './types.js'; + +/** + * Deterministic groundedness check over a generated response. + * + * The confidence signals we had before this module both ignored the *content* of + * the answer: `ResponseGenerator.assessConfidence` scored docs relevance and + * source count only, and the LLM scorer's rubric rewarded specificity ("cites + * specific features, APIs, or code patterns") without asking whether those + * citations were real. A confident fabrication therefore scored exactly as high + * as a cited answer — see CopilotKit/CopilotKit#6167, where the bot confirmed a + * bug it never reproduced and invented two CSS class names to explain it. + * + * This runs no model call. Everything here is a regex over the response plus a + * substring lookup against the sources the response was generated from, so it is + * cheap, deterministic, and unit-testable — the prompt rules in generator.ts are + * the request, this is the enforcement. + */ + +/** Claims of verification the bot cannot make: it has no repo, repro, or test run. */ +const UNVERIFIED_CLAIM_PATTERNS: Array<{ pattern: RegExp; label: string }> = [ + { pattern: /\bbug\s+confirmed\b/i, label: '"bug confirmed"' }, + { pattern: /\bconfirmed\s+(?:the\s+|this\s+|a\s+)?bug\b/i, label: 'claims the bug is confirmed' }, + { + pattern: /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, + label: 'asserts the report is a real bug', + }, + { pattern: /\bknown\s+(?:bug|issue|regression)\b/i, label: 'claims a known bug' }, + { pattern: /\broot\s+cause\s*(?:is\b|:)/i, label: 'asserts a root cause' }, + { pattern: /\bthe\s+fix\s+is\b/i, label: 'asserts the fix' }, + { + pattern: /\b(?:i|we)\s+(?:reproduced|replicated|verified|tested\s+this|ran\s+the\s+tests?)\b/i, + label: 'claims to have reproduced or tested', + }, +]; + +/** Hedge markers. A couple is honest; a pile means the answer is guesswork. */ +const HEDGE_PATTERNS: RegExp[] = [ + /\blikely\b/gi, + /\bprobably\b/gi, + /\bmight\s+be\b/gi, + /\bmay\s+vary\b/gi, + /\bshould\s+be\s+able\s+to\b/gi, + /\bi\s+(?:think|believe|suspect)\b/gi, + /\bnot\s+sure\b/gi, +]; + +/** + * CopilotKit-specific identifiers the response invents out of thin air. Scoped + * deliberately narrow — CSS classes and backticked tokens that carry our own + * name — so generic React vocabulary (`useRef`, `useLayoutEffect`) never trips + * it. `@copilotkit/*` package specifiers are excluded: those are stable public + * knowledge and routinely correct even when absent from the retrieved page. + */ +const CSS_CLASS_PATTERN = /\.(copilotKit[A-Za-z0-9_-]*)/g; +const BACKTICKED_PATTERN = /`([^`\n]{1,80})`/g; + +/** Hedges allowed before the density penalty starts. */ +const HEDGE_FREE_ALLOWANCE = 2; + +const PENALTY_PER_UNVERIFIED_CLAIM = 0.35; +const PENALTY_PER_UNSOURCED_IDENTIFIER = 0.15; +const PENALTY_PER_EXCESS_HEDGE = 0.03; + +/** Ceiling on the total deduction, so groundedness can't alone zero out a score. */ +export const MAX_GROUNDEDNESS_PENALTY = 0.6; + +/** + * Number of invented identifiers that, on its own, makes a response unsafe to + * publish. One could be a formatting artifact; two is a pattern of fabrication + * (#6167 shipped exactly two). + */ +const SUPPRESS_AT_UNSOURCED_IDENTIFIERS = 2; + +export interface GroundednessAssessment { + /** Amount to deduct from the confidence score (0 – MAX_GROUNDEDNESS_PENALTY). */ + penalty: number; + /** Verification claims the bot is not entitled to make. */ + unverifiedClaims: string[]; + /** CopilotKit identifiers named in the response but absent from every source. */ + unsourcedIdentifiers: string[]; + /** Total hedge markers found. */ + hedgeCount: number; + /** + * True when the response makes a claim we cannot stand behind. The caller is + * expected to withhold it from the public thread and escalate to a human + * instead — a lowered score alone does not stop a post. + */ + suppress: boolean; + /** Human-readable reasons, for logs and the dashboard. */ + reasons: string[]; +} + +/** + * Extract the CopilotKit-specific identifiers a response names. + * + * Exported for testing: the extraction rules are the part most likely to drift + * into false positives, so they're pinned directly. + */ +export function extractCopilotKitIdentifiers(response: string): string[] { + const found = new Set(); + + for (const match of response.matchAll(CSS_CLASS_PATTERN)) { + found.add(match[1]); + } + + for (const match of response.matchAll(BACKTICKED_PATTERN)) { + const token = match[1].trim(); + // Skip package specifiers and anything that isn't a bare identifier. + if (token.startsWith('@')) continue; + if (!/copilotkit/i.test(token)) continue; + if (!/^[.#]?[A-Za-z_$][\w$-]*$/.test(token)) continue; + found.add(token.replace(/^[.#]/, '')); + } + + return [...found]; +} + +/** + * Assess how well a generated response is supported by the sources it was + * generated from. Never throws — a malformed response yields a zero penalty + * rather than breaking the pipeline. + */ +export function assessGroundedness( + response: string, + sources: SearchResult[], +): GroundednessAssessment { + const empty: GroundednessAssessment = { + penalty: 0, + unverifiedClaims: [], + unsourcedIdentifiers: [], + hedgeCount: 0, + suppress: false, + reasons: [], + }; + + if (!response) return empty; + + const unverifiedClaims = UNVERIFIED_CLAIM_PATTERNS.filter(({ pattern }) => + pattern.test(response), + ).map(({ label }) => label); + + // Sources are searched as one haystack: an identifier documented on any + // retrieved page counts as grounded, regardless of which one. + const haystack = sources + .map((s) => `${s.title ?? ''}\n${s.content ?? ''}`) + .join('\n') + .toLowerCase(); + + const unsourcedIdentifiers = extractCopilotKitIdentifiers(response).filter( + (id) => !haystack.includes(id.toLowerCase()), + ); + + const hedgeCount = HEDGE_PATTERNS.reduce( + (count, pattern) => count + (response.match(pattern)?.length ?? 0), + 0, + ); + const excessHedges = Math.max(0, hedgeCount - HEDGE_FREE_ALLOWANCE); + + const penalty = Math.min( + unverifiedClaims.length * PENALTY_PER_UNVERIFIED_CLAIM + + unsourcedIdentifiers.length * PENALTY_PER_UNSOURCED_IDENTIFIER + + excessHedges * PENALTY_PER_EXCESS_HEDGE, + MAX_GROUNDEDNESS_PENALTY, + ); + + const suppress = + unverifiedClaims.length > 0 || + unsourcedIdentifiers.length >= SUPPRESS_AT_UNSOURCED_IDENTIFIERS; + + const reasons: string[] = []; + if (unverifiedClaims.length) { + reasons.push(`unverifiable claims: ${unverifiedClaims.join(', ')}`); + } + if (unsourcedIdentifiers.length) { + reasons.push(`identifiers absent from sources: ${unsourcedIdentifiers.join(', ')}`); + } + if (excessHedges > 0) { + reasons.push(`${hedgeCount} hedge markers`); + } + + return { penalty, unverifiedClaims, unsourcedIdentifiers, hedgeCount, suppress, reasons }; +} diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 8b777586..203efe79 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -2,6 +2,12 @@ export { PathfinderClient } from './pathfinder.js'; export { ResponseGenerator, GROUNDING_RULES, SYSTEM_PROMPT_PREFIX } from './generator.js'; export { ConfidenceScorer } from './confidence.js'; export type { ConfidenceAssessment } from './confidence.js'; +export { + assessGroundedness, + extractCopilotKitIdentifiers, + MAX_GROUNDEDNESS_PENALTY, +} from './groundedness.js'; +export type { GroundednessAssessment } from './groundedness.js'; export { TicketClassifier } from './classifier.js'; export { AI_DISCLAIMER, diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index bf8067e8..e0881a02 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -15,6 +15,7 @@ vi.mock('./config.js', () => ({ validateConfig: vi.fn(), })); +import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import { AIPipeline } from './pipeline.js'; import { ConfidenceLevel, TicketPriority, TicketType } from './types.js'; import type { SearchResult, GeneratedResponse } from './types.js'; @@ -214,6 +215,64 @@ describe('AIPipeline', () => { }, ); + // The scores above measure retrieval quality; these measure whether the + // answer stayed inside what was retrieved. Without this, a fabrication + // inherits the score of a good docs match (how #6167 got posted). + describe('groundedness', () => { + it('leaves a grounded response unpenalized and publishable', async () => { + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.groundedness.penalty).toBe(0); + expect(result.suppressed).toBe(false); + expect(result.confidenceScore).toBe(0.85); + }); + + it('deducts the penalty from the final score', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Override `.copilotKitInputControls` to fix it.', + }); + + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + // 0.85 (min of generator/scorer) − 0.15 for one invented identifier + expect(result.confidenceScore).toBeCloseTo(0.7, 5); + expect(result.groundedness.unsourcedIdentifiers).toEqual([ + 'copilotKitInputControls', + ]); + }); + + it('marks a response that confirms a bug as suppressed', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: '## Bug Confirmed: Cursor Jump\n\nRoot cause is a re-render.', + }); + + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.suppressed).toBe(true); + expect(result.groundedness.suppress).toBe(true); + expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); + }); + + // Positive feedback tunes how we weigh well-formed answers. It must not + // buy back a fabrication, so the penalty lands after calibration. + it('cannot be offset by positive feedback calibration', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Bug Confirmed. Root cause is a re-render. The fix is trivial.', + }); + + const withBoost = await pipeline.generateSupportResponse('q', { + source: 'github', + confidenceCalibration: 0.15, + }); + + expect(withBoost.suppressed).toBe(true); + expect(withBoost.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); + }); + }); + it('should handle Pathfinder failure gracefully', async () => { mockSearchDocs.mockRejectedValueOnce(new Error('MCP down')); diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index 58cf5c03..1f5dbac1 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -7,6 +7,7 @@ import type { SearchResult, } from './types.js'; import { ConfidenceLevel, classifyConfidence } from './types.js'; +import { assessGroundedness } from './groundedness.js'; import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import { PathfinderClient } from './pathfinder.js'; import { ResponseGenerator } from './generator.js'; @@ -27,6 +28,12 @@ import { validateConfig } from './config.js'; */ const DEGRADED_CONFIDENCE_CAP = AI_CONFIDENCE.HIGH_THRESHOLD - 0.01; +/** + * Highest score a suppressed (unpublishable) response may carry. Sits just below + * the escalation gate so a withheld answer always reads as needing a human. + */ +const SUPPRESSED_CONFIDENCE_CAP = AI_CONFIDENCE.ESCALATE - 0.01; + /** * Main entry point for the Outpost AI pipeline. * @@ -130,6 +137,27 @@ export class AIPipeline { Math.min(1, combinedConfidenceScore + calibration), ); + // Groundedness is deducted AFTER calibration so aggregate 👍/👎 feedback can + // never offset a fabrication: feedback tunes how we weigh a well-formed + // answer, it does not license an unsupported claim. + const groundedness = assessGroundedness(generatedResponse.text, searchResults); + if (groundedness.penalty > 0) { + finalConfidenceScore = Math.max(0, finalConfidenceScore - groundedness.penalty); + console.warn( + `[Pipeline] Groundedness penalty ${groundedness.penalty.toFixed(2)} — ${groundedness.reasons.join('; ')}`, + ); + } + + // A response we won't publish is not a confident one, whatever the + // retrieval scored. Clamp below the escalation gate so every downstream + // reader agrees: the disclaimer promises a human, the dashboard buckets it + // LOW, and the worker's score-based escalation fires on its own. The + // capped penalty alone can't guarantee this — a top score plus positive + // calibration lands exactly ON the gate, which does not escalate. + if (groundedness.suppress) { + finalConfidenceScore = Math.min(finalConfidenceScore, SUPPRESSED_CONFIDENCE_CAP); + } + // Safety cap: a DEGRADED confidence signal (LLM scorer unavailable → heuristic // fallback over Pathfinder's synthetic rank-scores) must never present as HIGH. // HIGH suppresses the disclaimer and is treated as authoritative, so an ungrounded @@ -169,6 +197,12 @@ export class AIPipeline { const latencyMs = Date.now() - startTime; + if (groundedness.suppress) { + console.warn( + `[Pipeline] Response withheld from public post — ${groundedness.reasons.join('; ')}`, + ); + } + return { response: generatedResponse.text, formatted, @@ -177,6 +211,8 @@ export class AIPipeline { searchResults, tokenUsage: totalTokenUsage, latencyMs, + groundedness, + suppressed: groundedness.suppress, }; } diff --git a/packages/outpost/ai/src/types.ts b/packages/outpost/ai/src/types.ts index 775d8dd0..fd024ede 100644 --- a/packages/outpost/ai/src/types.ts +++ b/packages/outpost/ai/src/types.ts @@ -4,6 +4,9 @@ import { AI_CONFIDENCE, TicketPriority, TicketType } from '@copilotkit/outpost/shared'; import type { PlatformTarget } from '@copilotkit/outpost/shared'; +// Type-only import — erased at build time, so the types.ts ↔ groundedness.ts +// cycle never exists at runtime. +import type { GroundednessAssessment } from './groundedness.js'; export { TicketPriority, TicketType } from '@copilotkit/outpost/shared'; export type { PlatformTarget } from '@copilotkit/outpost/shared'; @@ -207,4 +210,12 @@ export interface PipelineResult { tokenUsage: TokenUsage; /** End-to-end latency in milliseconds */ latencyMs: number; + /** Deterministic check of the response against its sources. */ + groundedness: GroundednessAssessment; + /** + * True when the response makes a claim we can't stand behind and must not be + * posted to the public thread. Callers escalate to a human instead. Mirrors + * `groundedness.suppress` — kept at the top level because it gates a post. + */ + suppressed: boolean; } diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index bd6823c4..15bb4516 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -137,6 +137,32 @@ const highConfidenceResult = { searchResults: [{ title: 'Getting Started', content: '...', score: 0.95 }], tokenUsage: { inputTokens: 100, outputTokens: 200 }, latencyMs: 1500, + groundedness: { + penalty: 0, + unverifiedClaims: [], + unsourcedIdentifiers: [], + hedgeCount: 0, + suppress: false, + reasons: [], + }, + suppressed: false, +}; + +/** A response the groundedness check refuses to publish (see #6167). */ +const suppressedResult = { + ...highConfidenceResult, + response: '## Bug Confirmed: Cursor Jump\n\nOverride `.copilotKitInputControls`.', + confidenceLevel: 'LOW', + confidenceScore: 0.32, + groundedness: { + penalty: 0.5, + unverifiedClaims: ['"bug confirmed"'], + unsourcedIdentifiers: ['copilotKitInputControls'], + hedgeCount: 0, + suppress: true, + reasons: ['unverifiable claims: "bug confirmed"'], + }, + suppressed: true, }; const mediumConfidenceResult = { @@ -283,6 +309,72 @@ describe('handleAiResponse', () => { ); }); + // A suppressed response is one the groundedness check found unsupportable. + // Confidence never gated the post-back, so these three behaviors are the + // whole point: nothing reaches the reporter, a human is pulled in, and the + // draft survives on the ticket for that human to edit. + describe('suppressed (ungrounded) responses', () => { + beforeEach(() => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(suppressedResult); + mockHasAdapter.mockReturnValue(true); + }); + + it('never posts to the source platform', async () => { + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data?.suppressed).toBe(true); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + + it('escalates to a human even though the score is above ESCALATE', async () => { + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + // 0.32 would escalate on score alone, so prove it escalates on + // suppression by raising the score above the gate. + mockPrismaJob.create.mockClear(); + mockGenerateSupportResponse.mockResolvedValue({ + ...suppressedResult, + confidenceScore: 0.95, + confidenceLevel: 'HIGH', + }); + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(result.data?.escalated).toBe(true); + expect(mockPrismaJob.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ type: 'ESCALATION' }), + }), + ); + const escalationCall = mockPrismaJob.create.mock.calls[0][0]; + expect(escalationCall.data.payload.reason).toContain('withheld'); + }); + + it('still persists the draft so a human can edit and send it', async () => { + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockPrismaMessage.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ type: 'BOT', isAiGenerated: true }), + }), + ); + expect(mockPrismaTicket.update).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + suggestedResponse: expect.any(String), + }), + }), + ); + }); + }); + it('does not escalate when confidence is above ESCALATE threshold', async () => { mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); mockGenerateSupportResponse.mockResolvedValue(mediumConfidenceResult); diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index aa639360..61279918 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -173,9 +173,22 @@ export async function handleAiResponse( }, }); - // 5b. Post the response back to the source platform + // 5b. Post the response back to the source platform. + // + // A suppressed response is one the groundedness check found unsupportable — + // it confirms a bug, asserts a root cause, or names identifiers absent from + // every retrieved source. Confidence alone never gated the post (it only + // picks the disclaimer and fires escalation), so a low score would not have + // stopped a fabrication from reaching a public thread. This does. The draft + // is still persisted above and lands on the ticket as suggestedResponse, so + // a human can edit and send it. const ticketSource = ticket.source as TicketSource; - if (process.env.SHADOW_MODE === 'true') { + if (pipelineResult.suppressed) { + console.warn( + `[AI Response] Withholding response for ticket ${ticketId} — ` + + `${pipelineResult.groundedness.reasons.join('; ')}. Escalating to a human.`, + ); + } else if (process.env.SHADOW_MODE === 'true') { try { await prisma.message.create({ data: { @@ -244,12 +257,16 @@ export async function handleAiResponse( await context.reportProgress(85); - // 6. If confidence is below the escalation threshold, enqueue ESCALATION - if (pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE) { + // 6. Enqueue ESCALATION when confidence is below threshold, or when the + // response was withheld — nothing reached the reporter in that case, so a + // human has to pick it up regardless of what the score says. + if (pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE || pipelineResult.suppressed) { try { await createJob(JobType.ESCALATION, { ticketId: ticket.id, - reason: `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation`, + reason: pipelineResult.suppressed + ? `AI response withheld (${pipelineResult.groundedness.reasons.join('; ')}) — needs a human answer` + : `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation`, }); } catch (error) { console.error( @@ -266,7 +283,8 @@ export async function handleAiResponse( console.log( `[AI Response] Ticket ${ticketId}: confidence=${pipelineResult.confidenceLevel} ` + - `(${(pipelineResult.confidenceScore * 100).toFixed(0)}%), latency=${pipelineResult.latencyMs}ms`, + `(${(pipelineResult.confidenceScore * 100).toFixed(0)}%), latency=${pipelineResult.latencyMs}ms` + + `${pipelineResult.suppressed ? ', response withheld' : ''}`, ); return { @@ -276,7 +294,10 @@ export async function handleAiResponse( confidenceLevel: pipelineResult.confidenceLevel, confidenceScore: pipelineResult.confidenceScore, latencyMs: pipelineResult.latencyMs, - escalated: pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE, + escalated: + pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE || + pipelineResult.suppressed, + suppressed: pipelineResult.suppressed, }, }; } From 28fb1fbbe22eee57681e4d29fed3ee9692ce7bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 28 Jul 2026 09:01:27 -0400 Subject: [PATCH 36/83] fix(ai): apply the groundedness penalty once, and stop suppressing negated claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes for #143. The penalty was deducted at two sites: generator.assessConfidence baked it into GeneratedResponse.confidenceScore, and the pipeline subtracted the same value again from the min() of the generator and scorer scores. Whenever the generator's already-penalized score was the lower of the two, it landed twice. The generator now assesses groundedness (it has the response and its sources in hand) and passes the result through on GeneratedResponse.groundedness without touching its own score, which goes back to measuring retrieval quality only. The pipeline is the single place that deducts, still after calibration. That also drops the redundant second assessGroundedness call per request. No existing test could have caught this: pipeline.test.ts injects a mocked ResponseGenerator, so assessConfidence never executes in any pipeline test. New pipeline-groundedness.test.ts runs the real generator against aimock and pins the arithmetic. Verified it fails on the pre-fix code (0.675 where 0.80 is expected) and passes after. Also from review: - Claim patterns were negation-blind. "This is not a known issue", "I can't determine what the root cause is", "I don't know what the fix is" all suppressed the response, and those are exactly the answers the prompt asks for. Claims are now checked per-occurrence against a same-sentence negation lookbehind, so a negated mention no longer excuses an assertive one elsewhere in the response. - CSS_CLASS_PATTERN is case-insensitive, so an invented `.copilotkit-input` or `.CopilotKitInput` is caught alongside `.copilotKitInput`. - Documented that identifier matching is substring rather than exact-token, and why the resulting false negatives are the safe direction. The suggestedResponse prefill item is filed separately — it is pre-existing, is not in this PR's diff, and its severity depends on a change that isn't here yet. --- packages/outpost/ai/src/generator.ts | 29 +-- packages/outpost/ai/src/groundedness.test.ts | 49 +++++ packages/outpost/ai/src/groundedness.ts | 47 ++++- .../ai/src/pipeline-groundedness.test.ts | 170 ++++++++++++++++++ packages/outpost/ai/src/pipeline.ts | 11 +- packages/outpost/ai/src/types.ts | 8 + 6 files changed, 299 insertions(+), 15 deletions(-) create mode 100644 packages/outpost/ai/src/pipeline-groundedness.test.ts diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index ea7c518a..cd3d6e38 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -127,9 +127,12 @@ export class ResponseGenerator { outputTokens: message.usage.output_tokens, }; - const confidenceScore = this.assessConfidence(sources, responseText); + const confidenceScore = this.assessConfidence(sources); const confidenceLevel = classifyConfidence(confidenceScore); const latencyMs = Date.now() - startTime; + // Assessed here (the response and its sources are both in hand) and + // applied by the pipeline — exactly once. + const groundedness = assessGroundedness(responseText, sources); return { text: responseText, @@ -140,6 +143,7 @@ export class ResponseGenerator { reasoning: `Based on ${sources.length} source(s) with avg relevance ${this.avgScore(sources).toFixed(2)}`, tokenUsage, latencyMs, + groundedness, degraded: false, }; } catch (error) { @@ -155,6 +159,8 @@ export class ResponseGenerator { reasoning: `Generation failed: ${error instanceof Error ? error.message : String(error)}`, tokenUsage: { inputTokens: 0, outputTokens: 0 }, latencyMs, + // The fallback copy is ours, not the model's — nothing to assess. + groundedness: assessGroundedness('', sources), degraded: true, }; } @@ -230,20 +236,23 @@ export class ResponseGenerator { return messages; } - private assessConfidence(sources: SearchResult[], response: string): number { + /** + * Score retrieval quality: how good the sources are, not what the response did + * with them. + * + * Deliberately does NOT deduct the groundedness penalty. This score feeds the + * pipeline's `min(generator, scorer)`, and the pipeline deducts afterwards — so + * subtracting here too charged the same penalty twice whenever this score was + * the lower of the two. The groundedness assessment travels alongside on + * `GeneratedResponse.groundedness` for the pipeline to apply once. + */ + private assessConfidence(sources: SearchResult[]): number { if (sources.length === 0) return 0.2; const avgRelevance = this.avgScore(sources); const sourceCountBonus = Math.min(sources.length * 0.05, 0.15); - // Base confidence on source quality + count - const retrievalScore = Math.min(avgRelevance + sourceCountBonus, 1.0); - - // Retrieval quality alone says nothing about whether the answer stayed - // inside those sources. Deduct for claims the response is not entitled - // to make, so a fabrication can't inherit a good docs match's score. - const { penalty } = assessGroundedness(response, sources); - return Math.max(0, retrievalScore - penalty); + return Math.min(avgRelevance + sourceCountBonus, 1.0); } private avgScore(sources: SearchResult[]): number { diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index 1696d5fe..7a0f703f 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -103,6 +103,55 @@ describe('assessGroundedness', () => { expect(result.suppress).toBe(true); }); + // Suppression is user-visible — the reporter gets the no-answer reply instead + // of a real one — so a false positive costs more than a missed one. These are + // all well-behaved responses of exactly the kind the prompt asks for. + describe('negated claims', () => { + it.each([ + 'This is not a known issue as far as the docs show.', + "I can't determine what the root cause is without reproducing it.", + "I don't know what the fix is — engineering will need to confirm.", + 'I have not reproduced this myself.', + "We haven't tested this against your version.", + "It's unclear whether this is a real bug or expected behavior.", + 'No bug confirmed here — the docs describe this as intended.', + ])('does not suppress %j', (response) => { + const result = assessGroundedness(response, CHAT_DOCS); + expect(result.unverifiedClaims).toEqual([]); + expect(result.suppress).toBe(false); + }); + + it('still catches an assertion in a later sentence', () => { + // The negation belongs to the first sentence only. + const result = assessGroundedness( + "I have not reproduced this. Root cause is a re-render on every keystroke.", + CHAT_DOCS, + ); + expect(result.unverifiedClaims).toContain('asserts a root cause'); + expect(result.suppress).toBe(true); + }); + + it('still catches an assertive occurrence when another is negated', () => { + const result = assessGroundedness( + "It's unclear whether the root cause is the layout. Bug confirmed regardless.", + CHAT_DOCS, + ); + expect(result.unverifiedClaims).toContain('"bug confirmed"'); + expect(result.suppress).toBe(true); + }); + }); + + it('flags invented class names regardless of case', () => { + for (const response of [ + 'Override `.copilotkit-input-controls` to fix it.', + 'Override `.CopilotKitInputControls` to fix it.', + ]) { + const result = assessGroundedness(response, CHAT_DOCS); + expect(result.unsourcedIdentifiers).toHaveLength(1); + expect(result.penalty).toBeGreaterThan(0); + } + }); + it('penalizes an identifier absent from every source', () => { const result = assessGroundedness( 'Override `.copilotKitInputControls` to force compact mode.', diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts index ca6008a7..172f52af 100644 --- a/packages/outpost/ai/src/groundedness.ts +++ b/packages/outpost/ai/src/groundedness.ts @@ -34,6 +34,33 @@ const UNVERIFIED_CLAIM_PATTERNS: Array<{ pattern: RegExp; label: string }> = [ }, ]; +/** + * Negation and uncertainty markers that flip a claim pattern's meaning. + * + * The patterns above match assertions, but the same words appear in exactly the + * responses the prompt asks for: "this is **not** a known issue", "I **can't** + * determine what the root cause is without reproducing it", "I **don't** know what + * the fix is". Suppression is user-visible (the reporter gets the no-answer reply + * instead of a real one), so a false positive costs more than a missed one. + * + * Anchored with `[^.!?]{0,N}$` so the marker has to sit in the SAME sentence as the + * claim — "That's confirmed. I have not tested it." must still count as a claim. + */ +const NEGATION_LOOKBEHIND = + /\b(?:not|never|cannot|can't|can not|won't|couldn't|don't|doesn't|didn't|unable|without|unclear|unsure|unconfirmed|no|nor|if|whether|maybe|possibly|suspect|guess)\b[^.!?]{0,80}$/i; + +/** How far back to look for a negation marker preceding a claim. */ +const NEGATION_WINDOW = 100; + +/** + * True when a claim match at `index` is negated or hedged by preceding text in the + * same sentence. + */ +function isNegated(response: string, index: number): boolean { + const before = response.slice(Math.max(0, index - NEGATION_WINDOW), index); + return NEGATION_LOOKBEHIND.test(before); +} + /** Hedge markers. A couple is honest; a pile means the answer is guesswork. */ const HEDGE_PATTERNS: RegExp[] = [ /\blikely\b/gi, @@ -52,7 +79,10 @@ const HEDGE_PATTERNS: RegExp[] = [ * it. `@copilotkit/*` package specifiers are excluded: those are stable public * knowledge and routinely correct even when absent from the retrieved page. */ -const CSS_CLASS_PATTERN = /\.(copilotKit[A-Za-z0-9_-]*)/g; +// Case-insensitive: an invented `.copilotkit-input` or `.CopilotKitInput` is just +// as ungrounded as `.copilotKitInput`, and the `/copilotkit/i` guard below already +// treats the name case-insensitively. +const CSS_CLASS_PATTERN = /\.(copilotkit[A-Za-z0-9_-]*)/gi; const BACKTICKED_PATTERN = /`([^`\n]{1,80})`/g; /** Hedges allowed before the density penalty starts. */ @@ -136,12 +166,21 @@ export function assessGroundedness( if (!response) return empty; - const unverifiedClaims = UNVERIFIED_CLAIM_PATTERNS.filter(({ pattern }) => - pattern.test(response), - ).map(({ label }) => label); + // A pattern counts only where it is actually asserted. Every occurrence is + // checked, so one negated mention doesn't excuse an assertive one elsewhere. + const unverifiedClaims = UNVERIFIED_CLAIM_PATTERNS.filter(({ pattern }) => { + const global = new RegExp(pattern.source, 'gi'); + return [...response.matchAll(global)].some((m) => !isNegated(response, m.index ?? 0)); + }).map(({ label }) => label); // Sources are searched as one haystack: an identifier documented on any // retrieved page counts as grounded, regardless of which one. + // + // Substring, not exact-token: an invented `copilotKitTextarea` counts as + // grounded if a source mentions `copilotKitTextareaWrapper`. Deliberate — the + // error goes toward NOT suppressing, and suppression is the user-visible + // outcome. Tighten to word boundaries only if fabrications start slipping + // through this way. const haystack = sources .map((s) => `${s.title ?? ''}\n${s.content ?? ''}`) .join('\n') diff --git a/packages/outpost/ai/src/pipeline-groundedness.test.ts b/packages/outpost/ai/src/pipeline-groundedness.test.ts new file mode 100644 index 00000000..c20f83d6 --- /dev/null +++ b/packages/outpost/ai/src/pipeline-groundedness.test.ts @@ -0,0 +1,170 @@ +import { describe, it, expect, vi, beforeAll, afterAll, beforeEach } from 'vitest'; +import { LLMock } from '@copilotkit/aimock'; + +vi.mock('./config.js', () => ({ + config: { + anthropicApiKey: 'test-key', + pathfinderMcpUrl: 'http://localhost:8787', + responseModel: 'claude-sonnet-4-6', + confidenceModel: 'claude-haiku-4-5-20251001', + classifierModel: 'claude-haiku-4-5-20251001', + sentimentModel: 'claude-haiku-4-5-20251001', + maxResponseTokens: 2048, + responseTemperature: 0.3, + confidence: { highThreshold: 0.8, mediumThreshold: 0.5 }, + pathfinder: { defaultLimit: 8, defaultMinScore: 0.3 }, + }, + validateConfig: vi.fn(), +})); + +import { AIPipeline } from './pipeline.js'; +import { ResponseGenerator } from './generator.js'; +import { ResponseFormatter } from './formatter.js'; +import { assessGroundedness } from './groundedness.js'; +import { ConfidenceLevel } from './types.js'; +import type { SearchResult } from './types.js'; + +/** + * The groundedness penalty must be applied exactly once, end to end. + * + * `pipeline.test.ts` cannot prove this: it injects a mocked `ResponseGenerator`, so + * `assessConfidence` — one of the two sites that used to deduct — never executes + * there. The double-application bug was invisible to the entire suite. These tests + * run the REAL generator against aimock so both sites are live, and assert the + * arithmetic that only holds when the deduction happens once. + */ + +let mock: LLMock; +let originalBaseUrl: string | undefined; + +beforeAll(async () => { + mock = new LLMock({ port: 0 }); + await mock.start(); + originalBaseUrl = process.env.ANTHROPIC_BASE_URL; + process.env.ANTHROPIC_BASE_URL = mock.url; +}); + +afterAll(async () => { + if (originalBaseUrl === undefined) { + delete process.env.ANTHROPIC_BASE_URL; + } else { + process.env.ANTHROPIC_BASE_URL = originalBaseUrl; + } + await mock.stop(); +}); + +beforeEach(() => { + mock.reset(); +}); + +/** avg(0.9, 0.85) = 0.875, + count bonus 0.10 → generator retrieval score 0.975. */ +const SOURCES: SearchResult[] = [ + { + title: 'CopilotChat', + content: 'Use the `input` prop to replace the chat input.', + score: 0.9, + category: 'copilotkit-docs', + }, + { + title: 'Styling', + content: 'Theme variables are documented here.', + score: 0.85, + category: 'copilotkit-docs', + }, +]; + +const SCORER_SCORE = 0.95; + +/** One invented identifier → penalty 0.15, and NOT suppressed (suppress needs 2). */ +const UNGROUNDED_RESPONSE = 'Override `.copilotKitInputControls` to force compact mode.'; + +function createPipeline() { + return new AIPipeline({ + pathfinder: { + searchDocs: vi.fn().mockResolvedValue(SOURCES), + searchAll: vi.fn().mockResolvedValue(SOURCES), + exploreDocs: vi.fn(), + queryKnowledgeBase: vi.fn(), + disconnect: vi.fn(), + } as never, + // The REAL generator — this is the point of the file. + generator: new ResponseGenerator({ apiKey: 'test-key' }), + confidenceScorer: { + score: vi.fn().mockResolvedValue({ + level: ConfidenceLevel.HIGH, + score: SCORER_SCORE, + reasoning: 'fixed for arithmetic', + tokenUsage: { inputTokens: 10, outputTokens: 5 }, + degraded: false, + }), + heuristicScore: vi.fn(), + } as never, + classifier: { classify: vi.fn(), heuristicClassify: vi.fn() } as never, + formatter: new ResponseFormatter(), + }); +} + +describe('groundedness penalty is applied exactly once', () => { + it('deducts the penalty a single time through the real generator', async () => { + mock.onMessage(/./, { + content: UNGROUNDED_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('how do I force compact mode?', { + source: 'github', + }); + + const { penalty } = assessGroundedness(UNGROUNDED_RESPONSE, SOURCES); + expect(penalty).toBeCloseTo(0.15, 5); + + // min(generator 0.975, scorer 0.95) = 0.95, minus one penalty of 0.15. + expect(result.confidenceScore).toBeCloseTo(SCORER_SCORE - penalty, 5); + + // The regression this file exists for: two deductions gave 0.80 − 0.15 = 0.65. + expect(result.confidenceScore).not.toBeCloseTo(SCORER_SCORE - penalty * 2, 5); + }); + + it('leaves the generator score free of the penalty so min() stays meaningful', async () => { + mock.onMessage(/./, { + content: UNGROUNDED_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const generated = await generator.generate({ question: 'q' }, SOURCES); + + // Retrieval quality only: avg 0.875 + 0.10 count bonus. + expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + // The assessment rides along for the pipeline to apply. + expect(generated.groundedness?.penalty).toBeCloseTo(0.15, 5); + }); + + it('still clamps a suppressed response below the escalation gate', async () => { + // Two invented identifiers → suppress. + mock.onMessage(/./, { + content: 'Override `.copilotKitInputControls` and `.copilotKitInputControlsExpanded`.', + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.suppressed).toBe(true); + expect(result.confidenceScore).toBeLessThan(0.4); + }); + + it('leaves a grounded response at full score', async () => { + mock.onMessage(/./, { + content: 'Use the `input` prop on CopilotChat to supply your own input component.', + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.groundedness.penalty).toBe(0); + expect(result.confidenceScore).toBeCloseTo(SCORER_SCORE, 5); + }); +}); diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index 1f5dbac1..15477f38 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -140,7 +140,16 @@ export class AIPipeline { // Groundedness is deducted AFTER calibration so aggregate 👍/👎 feedback can // never offset a fabrication: feedback tunes how we weigh a well-formed // answer, it does not license an unsupported claim. - const groundedness = assessGroundedness(generatedResponse.text, searchResults); + // + // This is the ONLY place the penalty is applied. The generator assesses + // groundedness (it has the response and its sources in hand) and passes the + // result through untouched — it must not deduct from its own + // `confidenceScore`, because that score feeds the `min()` above and the + // penalty would land twice. Recomputed here only if a caller injected a + // generator that doesn't supply one. + const groundedness = + generatedResponse.groundedness ?? + assessGroundedness(generatedResponse.text, searchResults); if (groundedness.penalty > 0) { finalConfidenceScore = Math.max(0, finalConfidenceScore - groundedness.penalty); console.warn( diff --git a/packages/outpost/ai/src/types.ts b/packages/outpost/ai/src/types.ts index fd024ede..139b556f 100644 --- a/packages/outpost/ai/src/types.ts +++ b/packages/outpost/ai/src/types.ts @@ -56,6 +56,14 @@ export interface GeneratedResponse { tokenUsage?: TokenUsage; /** End-to-end latency in milliseconds */ latencyMs?: number; + /** + * Groundedness of `text` against `sources`, assessed once here and consumed by + * the pipeline. The generator does NOT apply the penalty to `confidenceScore`: + * the pipeline is the single place that deducts, after feedback calibration. + * Subtracting in both places double-counted it, since `confidenceScore` feeds + * the pipeline's `min()` before its own deduction. + */ + groundedness?: GroundednessAssessment; } export interface TokenUsage { From e9ad3cbf779532c47acf3f51e4a60401d2f6128e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:19:58 -0400 Subject: [PATCH 37/83] fix(ai): rebuild the groundedness text analysis on normalize-then-split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The claim/negation core matched regexes against the raw response and then looked backwards over a fixed character window. That failed in both directions: it missed trailing negations, let a negation on one markdown bullet cancel an assertion on the next, could slice `cannot` into a bare `not`, and let hedges ("possibly the root cause is X") pose as negations. Alongside it, URLs were mined for identifiers, `sourceUrl` was never searched for grounding, dedup was case-sensitive while comparison was case-insensitive, and the penalty was charged per matched pattern rather than per claim. Replaced with normalize (strip URLs, neutralize "no doubt"-style intensifiers) → split into sentences on [.!?] and newlines → judge each sentence independently, checking for a genuine negator in both directions. Fenced code blocks are still in scope: #6167 put its invented class names in a ```css fence. Fixes, in order: trailing negation; hedges no longer negate; newline is a sentence boundary; NEGATION_WINDOW deleted rather than reconciled; URLs are never a source of identifiers; sourceUrl joins the grounding haystack; identifier dedup is case-folded (first spelling reported); claims are charged once per sentence per category, and the doc comment now matches. Also added `unknown`/`undetermined` as negators and made a `.` between digits a non-boundary so `v1.2.3` cannot strand a negation. Tests: new table-driven corpus organized by RESPONSE SHAPE, one row per shape with explicit expected suppression. 14 of its assertions failed against the old implementation and pass now. Fixed the vacuous 'picks up backticked CopilotKit identifiers' row, which asserted [] for a token containing no "copilotkit" and so passed via the wrong guard. Call-site enumeration (grep -rn across packages/ and apps/, excluding node_modules and dist): - assessGroundedness — unchanged signature and return shape. Callers: ai/src/pipeline.ts:152 (penalty subtracted, suppress gates the post), ai/src/generator.ts:135,163 (assessment rides along on GeneratedResponse), ai/src/index.ts:6 (re-export), ai/src/pipeline-groundedness.test.ts:119. All assumptions hold: same arity, same field names, penalty still bounded by MAX_GROUNDEDNESS_PENALTY, suppress still a boolean gate. Only the values change, and the pipeline/queue tests that pin those values (0.15 for one invented identifier, suppression on two) still pass. - extractCopilotKitIdentifiers — unchanged signature. Only call site outside its own tests is the re-export at ai/src/index.ts:7. Return is still string[] of bare identifiers; case-variant duplicates collapse and URL text no longer contributes, both strictly fewer entries. - MAX_GROUNDEDNESS_PENALTY — untouched value 0.6. Re-exported at ai/src/index.ts:8; assertions in groundedness.test.ts still hold. - GroundednessAssessment — all six fields unchanged. Consumed by ai/src/types.ts:9,66,222 and read in queue/src/handlers/ai-response.ts (reasons, suppress) — both still populated the same way. - SUPPRESS_AT_UNSOURCED_IDENTIFIERS — newly exported (was private, same value 2) and added to ai/src/index.ts so tests pin the bar by name. No pre-existing references to break. - NEGATION_LOOKBEHIND, NEGATION_WINDOW, isNegated — removed. grep -rn across packages/ and apps/ returns zero references anywhere. - New module-private helpers (stripUrls, splitSentences) and constants (NEGATOR_PATTERN, INTENSIFIER_PATTERN, INTENSIFIER_REPLACEMENT, URL_PATTERN, URL_PLACEHOLDER, SENTENCE_BOUNDARY, ClaimCategory) — not exported; grep confirms zero references outside groundedness.ts. Verified: vitest run --root packages/outpost → 874 passing across 56 files; tsc --noEmit for the ai project and the full package typecheck both clean. --- packages/outpost/ai/src/groundedness.test.ts | 224 ++++++++++++++++++- packages/outpost/ai/src/groundedness.ts | 203 +++++++++++++---- packages/outpost/ai/src/index.ts | 1 + 3 files changed, 383 insertions(+), 45 deletions(-) diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index 7a0f703f..21153549 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -3,6 +3,7 @@ import { assessGroundedness, extractCopilotKitIdentifiers, MAX_GROUNDEDNESS_PENALTY, + SUPPRESS_AT_UNSOURCED_IDENTIFIERS, } from './groundedness.js'; import type { SearchResult } from './types.js'; @@ -30,10 +31,13 @@ describe('extractCopilotKitIdentifiers', () => { expect(ids).toContain('copilotKitInputControlsExpanded'); }); - it('picks up backticked CopilotKit identifiers', () => { - expect(extractCopilotKitIdentifiers('Call `useCopilotChatInternals()` first.')).toEqual([]); - expect(extractCopilotKitIdentifiers('Call `useCopilotkitInternals` first.')).toContain( - 'useCopilotkitInternals', + it('picks up backticked CopilotKit identifiers but not call expressions', () => { + // Both tokens carry our name; only the bare identifier is a claim about an + // API surface we can check against the sources. `foo()` is prose-with-code, + // and the shape guard — not a missing "copilotkit" — is what rejects it. + expect(extractCopilotKitIdentifiers('Call `useCopilotKitInternals()` first.')).toEqual([]); + expect(extractCopilotKitIdentifiers('Call `useCopilotKitInternals` first.')).toContain( + 'useCopilotKitInternals', ); }); @@ -64,6 +68,22 @@ describe('extractCopilotKitIdentifiers', () => { ); expect(ids).toEqual(['copilotKitInput']); }); + + // Grounding compares case-insensitively, so dedup must too: otherwise two + // spellings of one invented name count as two fabrications and clear the + // suppression bar on their own. + it('deduplicates case variants, keeping the first spelling seen', () => { + const ids = extractCopilotKitIdentifiers('Use .copilotKitFoo, then .CopilotKitFoo.'); + expect(ids).toEqual(['copilotKitFoo']); + }); + + // The prompt tells the bot to cite docs URLs, so the hostname shows up in most + // good answers. A URL is never a declaration of an identifier. + it('never mines identifiers out of URLs', () => { + expect( + extractCopilotKitIdentifiers('See https://docs.copilotkit.ai/reference/chat for docs.'), + ).toEqual([]); + }); }); describe('assessGroundedness', () => { @@ -124,7 +144,7 @@ describe('assessGroundedness', () => { it('still catches an assertion in a later sentence', () => { // The negation belongs to the first sentence only. const result = assessGroundedness( - "I have not reproduced this. Root cause is a re-render on every keystroke.", + 'I have not reproduced this. Root cause is a re-render on every keystroke.', CHAT_DOCS, ); expect(result.unverifiedClaims).toContain('asserts a root cause'); @@ -261,3 +281,197 @@ describe('assessGroundedness', () => { expect(result.penalty).toBe(MAX_GROUNDEDNESS_PENALTY); }); }); + +/** + * Corpus organized by RESPONSE SHAPE, not by regex. + * + * The unit under test is "given a response that looks like THIS, do we post it?" — + * so each row names a shape a real answer takes (bare assertion, negated assertion, + * hedged assertion, markdown bullets, a cited docs URL, a code fence) and pins the + * publish/withhold decision for it. Rows are shape-complete rather than + * pattern-complete on purpose: it is the shapes that regress when the matching + * internals get rewritten. + */ +interface CorpusRow { + shape: string; + response: string; + sources?: SearchResult[]; + /** The decision that matters: does the reporter see this answer? */ + suppress: boolean; + /** Pinned only where the arithmetic is the point of the row. */ + penalty?: number; + unsourcedIdentifiers?: string[]; +} + +const urlSource = ( + sourceUrl: string, + title = 'Reference', + content = 'Docs page.', +): SearchResult => ({ + title, + content, + score: 0.9, + sourceUrl, +}); + +/** A source with no `sourceUrl` at all, so URL text cannot accidentally ground anything. */ +const NO_URL_DOCS: SearchResult[] = [ + { title: 'CopilotChat', content: 'CopilotChat renders a chat window.', score: 0.9 }, +]; + +const CORPUS: CorpusRow[] = [ + { + shape: 'bare assertion of a root cause', + response: 'Root cause is a re-render on every keystroke.', + suppress: true, + penalty: 0.35, + }, + { + shape: 'assertion cancelled by a negation BEFORE it', + response: 'I cannot tell what the root cause is from the docs alone.', + suppress: false, + penalty: 0, + }, + { + shape: 'assertion cancelled by a negation AFTER it in the same sentence', + response: 'The root cause is not obvious from the docs.', + suppress: false, + penalty: 0, + }, + { + // Version numbers must not fragment the sentence, or the negation lands in + // a different fragment than the claim and a good answer gets withheld. + shape: 'negated assertion whose sentence contains a version number', + response: "I can't reproduce on 1.2.3, so the root cause is a mystery.", + suppress: false, + penalty: 0, + }, + { + shape: 'assertion cancelled by a trailing uncertainty marker', + response: 'What the root cause is remains unclear.', + suppress: false, + penalty: 0, + }, + { + // Hedging does not buy the right to assert. The hedge-density penalty is a + // separate, softer signal; it must not double as an assertion escape hatch. + shape: 'hedged assertion ("possibly ...")', + response: 'Possibly the root cause is a re-render.', + suppress: true, + penalty: 0.35, + }, + { + shape: 'assertion in the sentence AFTER a negated one', + response: 'I have not reproduced this. Root cause is a re-render.', + suppress: true, + penalty: 0.35, + }, + { + // A newline ends a thought as firmly as a period does; a negation on the + // previous bullet says nothing about this one. + shape: 'negation and assertion on separate markdown bullets', + response: '- No workaround exists yet\n- Root cause is a re-render on every keystroke', + suppress: true, + penalty: 0.35, + }, + { + shape: 'cites a docs URL whose path documents the identifier it names', + response: + 'Wrap your app in `CopilotKitProvider` — see ' + + 'https://docs.copilotkit.ai/reference/components/CopilotKitProvider.', + sources: [urlSource('https://docs.copilotkit.ai/reference/components/CopilotKitProvider')], + suppress: false, + penalty: 0, + unsourcedIdentifiers: [], + }, + { + // The prompt asks for docs links, so the hostname appears in most good + // answers. It must never register as an identifier of its own. + shape: 'cites a docs URL absent from the sources', + response: + 'Full details live at https://docs.copilotkit.ai/reference/components/chat/CopilotChat.', + sources: NO_URL_DOCS, + suppress: false, + penalty: 0, + unsourcedIdentifiers: [], + }, + { + // ...and the reverse: a URL in the RESPONSE cannot launder an invented name. + shape: 'names an identifier that exists only in a URL the response itself invented', + response: + 'See https://docs.copilotkit.ai/reference/copilotKitPhantomHook for ' + + '`copilotKitPhantomHook`.', + suppress: false, + penalty: 0.15, + unsourcedIdentifiers: ['copilotKitPhantomHook'], + }, + { + // #6167 put its invented classes inside a ```css fence. Fences are where + // fabricated identifiers live, so they stay in scope. + shape: 'invented class names inside a ```css fence', + response: [ + '```css', + '.copilotKitGhostA { color: red; }', + '.copilotKitGhostB { color: blue; }', + '```', + ].join('\n'), + suppress: true, + unsourcedIdentifiers: ['copilotKitGhostA', 'copilotKitGhostB'], + }, + { + shape: 'one invented identifier written in two casings', + response: 'Override `.copilotKitFoo` and `.CopilotKitFoo` to fix it.', + suppress: false, + penalty: 0.15, + unsourcedIdentifiers: ['copilotKitFoo'], + }, + { + // Two claim patterns fire on this one sentence; it is still one claim. + shape: '"known bug" assertion matching several patterns at once', + response: 'This is a known bug.', + suppress: true, + penalty: 0.35, + }, + { + shape: '"no doubt" used as an intensifier, not a negation', + response: 'There is no doubt this is a real bug.', + suppress: true, + penalty: 0.35, + }, + { + shape: '"no question about it" used as an intensifier', + response: 'No question about it, bug confirmed.', + suppress: true, + penalty: 0.35, + }, + { + shape: 'grounded answer that asserts nothing it cannot support', + response: + 'You can replace the chat input with the `input` prop on the `CopilotChat` ' + + 'component. That keeps your own state, so you control the cursor.', + suppress: false, + penalty: 0, + unsourcedIdentifiers: [], + }, +]; + +describe('assessGroundedness response-shape corpus', () => { + it.each(CORPUS)('$shape', ({ response, sources, suppress, penalty, unsourcedIdentifiers }) => { + const result = assessGroundedness(response, sources ?? CHAT_DOCS); + + expect(result.suppress).toBe(suppress); + if (penalty !== undefined) expect(result.penalty).toBeCloseTo(penalty, 5); + if (unsourcedIdentifiers !== undefined) { + expect(result.unsourcedIdentifiers).toEqual(unsourcedIdentifiers); + } + }); + + it('pins the suppression bar to the exported threshold', () => { + const twoInvented = assessGroundedness( + 'Override `.copilotKitGhostA` and `.copilotKitGhostB`.', + CHAT_DOCS, + ); + expect(twoInvented.unsourcedIdentifiers).toHaveLength(SUPPRESS_AT_UNSOURCED_IDENTIFIERS); + expect(twoInvented.suppress).toBe(true); + }); +}); diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts index 172f52af..5811e68b 100644 --- a/packages/outpost/ai/src/groundedness.ts +++ b/packages/outpost/ai/src/groundedness.ts @@ -15,27 +15,66 @@ import type { SearchResult } from './types.js'; * substring lookup against the sources the response was generated from, so it is * cheap, deterministic, and unit-testable — the prompt rules in generator.ts are * the request, this is the enforcement. + * + * Shape of the analysis: **normalize, then split, then judge each sentence.** + * Earlier revisions matched claim patterns against the whole response and then + * looked backwards over a fixed character window for a negation. That failed in + * both directions — it missed trailing negations ("the root cause is not + * obvious"), let a negation on one markdown bullet cancel an assertion on the + * next, and could slice `cannot` into a bare `not`. Sentences are the unit a + * negation actually scopes over, so we cut the text into them first and ask each + * one, independently, "is this asserted here?". */ -/** Claims of verification the bot cannot make: it has no repo, repro, or test run. */ -const UNVERIFIED_CLAIM_PATTERNS: Array<{ pattern: RegExp; label: string }> = [ - { pattern: /\bbug\s+confirmed\b/i, label: '"bug confirmed"' }, - { pattern: /\bconfirmed\s+(?:the\s+|this\s+|a\s+)?bug\b/i, label: 'claims the bug is confirmed' }, +/** The kind of unsupportable claim a pattern detects. Several patterns can share one. */ +type ClaimCategory = 'confirmation' | 'bug-validity' | 'root-cause' | 'fix' | 'reproduction'; + +/** + * Claims of verification the bot cannot make: it has no repo, repro, or test run. + * + * `category` exists so overlapping wordings are charged once. "This is a known + * bug" trips both the real-bug pattern and the known-bug pattern; it is still one + * claim, and billing it twice made a single sentence cost more than two distinct + * fabrications. + */ +const UNVERIFIED_CLAIM_PATTERNS: Array<{ + pattern: RegExp; + label: string; + category: ClaimCategory; +}> = [ + { pattern: /\bbug\s+confirmed\b/i, label: '"bug confirmed"', category: 'confirmation' }, { - pattern: /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, + pattern: /\bconfirmed\s+(?:the\s+|this\s+|a\s+)?bug\b/i, + label: 'claims the bug is confirmed', + category: 'confirmation', + }, + { + pattern: + /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, label: 'asserts the report is a real bug', + category: 'bug-validity', + }, + { + pattern: /\bknown\s+(?:bug|issue|regression)\b/i, + label: 'claims a known bug', + category: 'bug-validity', }, - { pattern: /\bknown\s+(?:bug|issue|regression)\b/i, label: 'claims a known bug' }, - { pattern: /\broot\s+cause\s*(?:is\b|:)/i, label: 'asserts a root cause' }, - { pattern: /\bthe\s+fix\s+is\b/i, label: 'asserts the fix' }, { - pattern: /\b(?:i|we)\s+(?:reproduced|replicated|verified|tested\s+this|ran\s+the\s+tests?)\b/i, + pattern: /\broot\s+cause\s*(?:is\b|:)/i, + label: 'asserts a root cause', + category: 'root-cause', + }, + { pattern: /\bthe\s+fix\s+is\b/i, label: 'asserts the fix', category: 'fix' }, + { + pattern: + /\b(?:i|we)\s+(?:reproduced|replicated|verified|tested\s+this|ran\s+the\s+tests?)\b/i, label: 'claims to have reproduced or tested', + category: 'reproduction', }, ]; /** - * Negation and uncertainty markers that flip a claim pattern's meaning. + * Words that genuinely reverse a claim inside its own sentence. * * The patterns above match assertions, but the same words appear in exactly the * responses the prompt asks for: "this is **not** a known issue", "I **can't** @@ -43,32 +82,59 @@ const UNVERIFIED_CLAIM_PATTERNS: Array<{ pattern: RegExp; label: string }> = [ * the fix is". Suppression is user-visible (the reporter gets the no-answer reply * instead of a real one), so a false positive costs more than a missed one. * - * Anchored with `[^.!?]{0,N}$` so the marker has to sit in the SAME sentence as the - * claim — "That's confirmed. I have not tested it." must still count as a claim. + * Hedges — `maybe`, `possibly`, `suspect`, `guess` — are deliberately NOT here. + * They do not reverse a claim, they only soften its delivery, and "possibly the + * root cause is X" is precisely the confident-guess shape this gate exists to + * catch. Hedging is priced separately by the hedge-density penalty below. + * + * `no` stays, because "no bug confirmed here" is a real negation — but see + * INTENSIFIER_PATTERN for the phrases where it means the opposite. */ -const NEGATION_LOOKBEHIND = - /\b(?:not|never|cannot|can't|can not|won't|couldn't|don't|doesn't|didn't|unable|without|unclear|unsure|unconfirmed|no|nor|if|whether|maybe|possibly|suspect|guess)\b[^.!?]{0,80}$/i; +const NEGATOR_PATTERN = + /\b(?:not|never|cannot|can\s+not|unable|without|unclear|unsure|unconfirmed|unknown|undetermined|no|nor|none|neither)\b|n['’]t\b/i; -/** How far back to look for a negation marker preceding a claim. */ -const NEGATION_WINDOW = 100; +/** + * Phrases where a negator is actually an intensifier: "there is **no doubt** this + * is a real bug" asserts harder than the plain sentence does. Neutralized before + * the negator scan so they cannot wave a claim through. + */ +const INTENSIFIER_PATTERN = /\b(?:no|without|beyond)\s+(?:a\s+)?(?:doubt|question)s?\b/gi; +const INTENSIFIER_REPLACEMENT = 'certainly'; /** - * True when a claim match at `index` is negated or hedged by preceding text in the - * same sentence. + * URLs are replaced with this before any analysis. + * + * Two reasons. Identifiers: the prompt actively instructs the bot to cite docs + * URLs, so `https://docs.copilotkit.ai/...` shows up in most *good* answers, and + * reading the hostname as a declaration of `.copilotkit` invented a fabrication + * out of a correct citation. Sentences: a URL is full of `.` and `?`, which would + * shred one sentence into several. The placeholder carries no `.`, so it is inert + * on both counts. */ -function isNegated(response: string, index: number): boolean { - const before = response.slice(Math.max(0, index - NEGATION_WINDOW), index); - return NEGATION_LOOKBEHIND.test(before); -} +const URL_PATTERN = /\b(?:https?:\/\/|www\.)[^\s<>()[\]{}"'`]+/gi; +const URL_PLACEHOLDER = ' [url] '; + +/** + * Sentence boundary: terminal punctuation OR a line break. A newline ends a + * thought as firmly as a period does — markdown answers are mostly bullets, and a + * negation on one bullet says nothing about the next. + * + * A `.` between digits is not a boundary, so `v1.2.3` stays inside its sentence. + * Splitting a version number would strand the negation ("I can't reproduce this on + * 1.2.3") in a different fragment from the claim, which suppresses a good answer. + */ +const SENTENCE_BOUNDARY = /(? sentence.trim()) + .filter((sentence) => sentence.length > 0); +} + /** * Extract the CopilotKit-specific identifiers a response names. * * Exported for testing: the extraction rules are the part most likely to drift * into false positives, so they're pinned directly. + * + * Dedup is case-folded to match the case-insensitive grounding comparison in + * `assessGroundedness` — otherwise `.copilotKitFoo` plus `.CopilotKitFoo` counts + * as two fabrications and clears the suppression bar by itself. The first + * spelling seen is the one reported, so log lines quote the response. */ export function extractCopilotKitIdentifiers(response: string): string[] { - const found = new Set(); + const found = new Map(); + const remember = (identifier: string): void => { + const key = identifier.toLowerCase(); + if (!found.has(key)) found.set(key, identifier); + }; - for (const match of response.matchAll(CSS_CLASS_PATTERN)) { - found.add(match[1]); + const text = stripUrls(response); + + for (const match of text.matchAll(CSS_CLASS_PATTERN)) { + remember(match[1]); } - for (const match of response.matchAll(BACKTICKED_PATTERN)) { + for (const match of text.matchAll(BACKTICKED_PATTERN)) { const token = match[1].trim(); // Skip package specifiers and anything that isn't a bare identifier. if (token.startsWith('@')) continue; if (!/copilotkit/i.test(token)) continue; if (!/^[.#]?[A-Za-z_$][\w$-]*$/.test(token)) continue; - found.add(token.replace(/^[.#]/, '')); + remember(token.replace(/^[.#]/, '')); } - return [...found]; + return [...found.values()]; } /** @@ -166,15 +265,39 @@ export function assessGroundedness( if (!response) return empty; - // A pattern counts only where it is actually asserted. Every occurrence is - // checked, so one negated mention doesn't excuse an assertive one elsewhere. - const unverifiedClaims = UNVERIFIED_CLAIM_PATTERNS.filter(({ pattern }) => { - const global = new RegExp(pattern.source, 'gi'); - return [...response.matchAll(global)].some((m) => !isNegated(response, m.index ?? 0)); - }).map(({ label }) => label); + const normalized = stripUrls(response).replace(INTENSIFIER_PATTERN, INTENSIFIER_REPLACEMENT); + + // A claim counts where it is asserted, sentence by sentence: a sentence with no + // genuine negator anywhere in it — before OR after the matched wording — + // asserts what it says. One negated mention therefore does not excuse an + // assertive one elsewhere, and an assertion is not excused by a negation that + // belongs to a neighbouring sentence. + // + // Charging is once per sentence per claim CATEGORY, so overlapping wordings of + // one accusation ("this is a known bug") cost one claim, not two. + const unverifiedClaims: string[] = []; + const seenLabels = new Set(); + let chargeableClaims = 0; + + for (const sentence of splitSentences(normalized)) { + if (NEGATOR_PATTERN.test(sentence)) continue; + + const categories = new Set(); + for (const { pattern, label, category } of UNVERIFIED_CLAIM_PATTERNS) { + if (!pattern.test(sentence)) continue; + categories.add(category); + if (!seenLabels.has(label)) { + seenLabels.add(label); + unverifiedClaims.push(label); + } + } + chargeableClaims += categories.size; + } // Sources are searched as one haystack: an identifier documented on any - // retrieved page counts as grounded, regardless of which one. + // retrieved page counts as grounded, regardless of which one. `sourceUrl` is + // part of the haystack — a response naming the very page it was handed + // (".../reference/components/CopilotKitProvider") is citing, not inventing. // // Substring, not exact-token: an invented `copilotKitTextarea` counts as // grounded if a source mentions `copilotKitTextareaWrapper`. Deliberate — the @@ -182,7 +305,7 @@ export function assessGroundedness( // outcome. Tighten to word boundaries only if fabrications start slipping // through this way. const haystack = sources - .map((s) => `${s.title ?? ''}\n${s.content ?? ''}`) + .map((s) => `${s.title ?? ''}\n${s.content ?? ''}\n${s.sourceUrl ?? ''}`) .join('\n') .toLowerCase(); @@ -191,13 +314,13 @@ export function assessGroundedness( ); const hedgeCount = HEDGE_PATTERNS.reduce( - (count, pattern) => count + (response.match(pattern)?.length ?? 0), + (count, pattern) => count + (normalized.match(pattern)?.length ?? 0), 0, ); const excessHedges = Math.max(0, hedgeCount - HEDGE_FREE_ALLOWANCE); const penalty = Math.min( - unverifiedClaims.length * PENALTY_PER_UNVERIFIED_CLAIM + + chargeableClaims * PENALTY_PER_UNVERIFIED_CLAIM + unsourcedIdentifiers.length * PENALTY_PER_UNSOURCED_IDENTIFIER + excessHedges * PENALTY_PER_EXCESS_HEDGE, MAX_GROUNDEDNESS_PENALTY, diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 203efe79..6a94e2a3 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -6,6 +6,7 @@ export { assessGroundedness, extractCopilotKitIdentifiers, MAX_GROUNDEDNESS_PENALTY, + SUPPRESS_AT_UNSOURCED_IDENTIFIERS, } from './groundedness.js'; export type { GroundednessAssessment } from './groundedness.js'; export { TicketClassifier } from './classifier.js'; From da48fc1a03cf161b5fa184814fa10e07a582b95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:51:42 -0400 Subject: [PATCH 38/83] fix(ai): gate groundedness suppression on the objective signal only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Withholding a reply was driven by regex-parsed English: claim phrases plus a negation detector. That decision has now failed three times in three different ways. The character-window version suppressed a good answer ("The root cause is not obvious"); the sentence-scoped negator list waved fabrications through because any negator anywhere in the sentence excused the claim: "This is a known issue with no workaround." -> suppress false "I cannot reproduce it, but the root cause is a re-render." -> suppress false "Bug confirmed, though I have no repro steps." -> suppress false, penalty 0 Natural-language negation is not regex-tractable, so it stops gating a user-visible publish/withhold decision. Restructure: 1. `suppress` is now driven ONLY by the objective signal — CopilotKit identifiers named in the response but absent from every retrieved source (`unsourcedIdentifiers.length >= SUPPRESS_AT_UNSOURCED_IDENTIFIERS`). That is checkable against the sources we handed the model; no English is parsed. 2. Claim phrases are penalty-only. Still detected, still charged `PENALTY_PER_UNVERIFIED_CLAIM`, still reported on `unverifiedClaims` and `reasons` — but they contribute nothing to `suppress`. The penalty still drags the score under the escalation gate, so a human is pulled in and the disclaimer still lands. The consequence of a misread is a lower score, never a withheld reply. 3. The negation machinery is deleted entirely — sentence splitter (`SENTENCE_BOUNDARY`, `splitSentences`), negator list (`NEGATOR_PATTERN`), and the intensifier carve-outs (`INTENSIFIER_PATTERN`, `INTENSIFIER_REPLACEMENT`). Nothing needs it once claims are penalty-only: a false positive on "the root cause is not obvious" costs 0.35 of confidence instead of the reporter's answer. Removing it removes the whole recurring class of bug rather than tuning it a fourth time. 4. Identifier extraction gaps closed, since this signal now carries the whole gate: - Scheme-less hostnames are stripped (`BARE_HOST_PATTERN`). `URL_PATTERN` only caught `https://` / `www.`, so `docs.copilotkit.ai/reference` still yielded the phantom identifier `copilotkit` — which under the new contract is not merely a wrong penalty, it is two phantoms away from withholding a correct citation. Anchored on a known TLD so `1.2.3` and `.copilotKitInput.copilotKitInputExpanded` are not mistaken for hosts. - `reasons` now agree with what was charged. `unverifiedClaims` was deduped globally while charging counted per-sentence categories, so the logged basis understated the deduction. Charging is now one per claim CATEGORY over the response and the reported labels ARE the charged ones — `penalty == unverifiedClaims.length * PENALTY_PER_UNVERIFIED_CLAIM` (pre-cap). Pattern order was adjusted so the more specific wording supplies the label ("known issue" reads as "claims a known bug"). - The bare-identifier guard was widened (`identifierSegments`): call expressions (`useCopilotKitFoo()`, `useCopilotKitFoo({...})`), JSX (``, ``, ``) and dotted member forms (`window.copilotKitFoo` -> `copilotKitFoo`) all count now; a fabrication must not escape the gate on syntax alone. `@copilotkit/*` package specifiers, including subpaths, stay excluded. Results are ordered by position in the response so logs read like the answer. 5. Public API preserved: `assessGroundedness`, `extractCopilotKitIdentifiers`, `MAX_GROUNDEDNESS_PENALTY`, `SUPPRESS_AT_UNSOURCED_IDENTIFIERS` and every `GroundednessAssessment` field. Module stays deterministic — no model calls. Known edge (unchanged code, stated for the record): a claim-only response that starts at a perfect score AND carries the maximum positive feedback calibration lands exactly ON `AI_CONFIDENCE.ESCALATE` (1.0 - 0.6 cap = 0.4) rather than below it, so it is not escalated by score. Previously the suppression clamp in pipeline.ts covered that case for claim wording; it now covers only the identifier signal. Pinned by pipeline.test.ts "cannot be offset by positive feedback calibration". Raising `MAX_GROUNDEDNESS_PENALTY` or clamping on `unverifiedClaims.length > 0` in pipeline.ts would close it; both are behavior changes outside this fix. ## Call-site enumeration `grep -rn packages apps`, excluding node_modules and dist. REMOVED (zero remaining references, verified by grep): - `NEGATOR_PATTERN` — (no references) - `INTENSIFIER_PATTERN` — (no references) - `INTENSIFIER_REPLACEMENT` — (no references) - `SENTENCE_BOUNDARY` — (no references) - `splitSentences` — (no references) - `chargeableClaims` (local) — (no references) All five were module-private in groundedness.ts and never exported from ai/src/index.ts, so removal is invisible outside the file. ADDED (module-private, no external call sites by design): - `BARE_HOST_PATTERN`, `JSX_WRAPPER`, `CALL_EXPRESSION`, `IDENTIFIER_PATH`, `identifierSegments` — referenced only inside groundedness.ts; grep finds no other site. Behavior is pinned through the two exported functions. CHANGED — `assessGroundedness(response, sources)` (signature and return type unchanged; `suppress` and `unverifiedClaims` semantics changed): - packages/outpost/ai/src/generator.ts:6,135 — assesses and passes the result through on `GeneratedResponse.groundedness` without reading `suppress` or `unverifiedClaims`. Assumption holds: it only forwards the object. - packages/outpost/ai/src/generator.ts:163 — fallback path calls with `''`; the empty-response short circuit is untouched, still an all-zero assessment. - packages/outpost/ai/src/pipeline.ts:10,152 — recompute path when a caller injects a generator that supplies no assessment. Assumption holds; same signature, same field set. - packages/outpost/ai/src/pipeline.ts:153-158 — applies `penalty` once. Holds: penalty is still 0..MAX_GROUNDEDNESS_PENALTY, and claim wording still contributes to it, so the escalation path for a misworded answer is intact. - packages/outpost/ai/src/pipeline.ts:166-168 — clamps a suppressed response to `SUPPRESSED_CONFIDENCE_CAP`. Still correct, now narrower: it fires on the identifier signal only. See "Known edge" above. - packages/outpost/ai/src/pipeline.ts:209-213,223-224 — logs `reasons` and publishes `suppressed: groundedness.suppress`. Holds; `reasons` is unchanged in shape and now strictly more accurate about the penalty. - packages/outpost/ai/src/pipeline-groundedness.test.ts:23,119 — arithmetic fixture uses a one-invented-identifier response (penalty 0.15, not suppressed) and a two-identifier response for the clamp. Both assumptions hold under the new gate; file needed no edit and passes unchanged. - packages/outpost/ai/src/index.ts:6 — re-export only. CHANGED — `extractCopilotKitIdentifiers(response)` (widened; signature and return type unchanged): - packages/outpost/ai/src/index.ts:7 — re-export only. No other production call site; it is used inside groundedness.ts and by tests. Assumption holds: still `(string) => string[]`, still deduped case-insensitively, now ordered by position and inclusive of call/JSX/dotted spellings. UNCHANGED exports, confirmed still referenced and still valid: - `MAX_GROUNDEDNESS_PENALTY` — ai/src/index.ts:8 (re-export), value 0.6 kept. - `SUPPRESS_AT_UNSOURCED_IDENTIFIERS` — ai/src/index.ts:9 (re-export), value 2 kept; it is now the sole input to `suppress`. - `GroundednessAssessment` — ai/src/index.ts:11, ai/src/types.ts:9,66,222. Field set identical (`penalty`, `unverifiedClaims`, `unsourcedIdentifiers`, `hedgeCount`, `suppress`, `reasons`), so both `GeneratedResponse.groundedness` and `PipelineResult.groundedness` are unaffected. types.ts:226's comment ("`groundedness.suppress` … gates a post") is still accurate. Queue handler — no source change needed, assumptions verified: - packages/outpost/queue/src/handlers/ai-response.ts:186 — skips the public post on `pipelineResult.suppressed`. Holds; suppressed now means "named identifiers no source contains", which is exactly the unpublishable case. - .../ai-response.ts:189,268 — renders `groundedness.reasons` into the internal note and the escalation reason. Holds; `reasons` still populated for both claims and identifiers. - .../ai-response.ts:263,267,287,299,300 — escalate on `score < ESCALATE || suppressed`; unchanged semantics. - `npx tsc --project queue/tsconfig.json --noEmit` is clean. Tests touched: - packages/outpost/ai/src/pipeline.test.ts — "marks a response that confirms a bug as suppressed" replaced by "penalizes a bug-confirming response into escalation without withholding it" plus "marks a response naming identifiers absent from the sources as suppressed"; the calibration test now pins 0.85 + 0.15 - 0.6 = 0.4 and a new sibling proves a boost cannot lift a suppressed response over the gate. - packages/outpost/queue/src/__tests__/ai-response.test.ts — the hand-built `suppressedResult` fixture described an impossible assessment under the new contract (suppress: true with one unsourced identifier). Given the two invented class names #6167 actually shipped, plus the matching reasons. Verification: `npx vitest run --root packages/outpost --reporter=dot` → 56 files / 883 tests green. `npx tsc --noEmit -p packages/outpost/ai` clean; db, ai and queue projects all compile. --- packages/outpost/ai/src/groundedness.test.ts | 348 +++++++++++++----- packages/outpost/ai/src/groundedness.ts | 258 +++++++------ packages/outpost/ai/src/pipeline.test.ts | 68 +++- .../queue/src/__tests__/ai-response.test.ts | 21 +- 4 files changed, 481 insertions(+), 214 deletions(-) diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index 21153549..3cb4bf14 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -31,14 +31,38 @@ describe('extractCopilotKitIdentifiers', () => { expect(ids).toContain('copilotKitInputControlsExpanded'); }); - it('picks up backticked CopilotKit identifiers but not call expressions', () => { - // Both tokens carry our name; only the bare identifier is a claim about an - // API surface we can check against the sources. `foo()` is prose-with-code, - // and the shape guard — not a missing "copilotkit" — is what rejects it. - expect(extractCopilotKitIdentifiers('Call `useCopilotKitInternals()` first.')).toEqual([]); - expect(extractCopilotKitIdentifiers('Call `useCopilotKitInternals` first.')).toContain( + // The identifier signal is now the ONLY thing that can withhold a response, so + // a name written as a call, as JSX, or behind a dot has to count the same as the + // bare spelling — otherwise a fabrication escapes the gate on syntax alone. + it('picks up a backticked identifier written as a call expression', () => { + expect(extractCopilotKitIdentifiers('Call `useCopilotKitInternals()` first.')).toEqual([ 'useCopilotKitInternals', - ); + ]); + expect( + extractCopilotKitIdentifiers('Call `useCopilotKitInternals({ debug: true })` first.'), + ).toEqual(['useCopilotKitInternals']); + }); + + it('picks up a backticked identifier written as JSX', () => { + expect(extractCopilotKitIdentifiers('Wrap it in ``.')).toEqual([ + 'CopilotKitGhostPanel', + ]); + expect(extractCopilotKitIdentifiers('Close with ``.')).toEqual([ + 'CopilotKitGhostPanel', + ]); + expect(extractCopilotKitIdentifiers('Open with ``.')).toEqual([ + 'CopilotKitGhostPanel', + ]); + }); + + it('picks up the CopilotKit-named segments of a dotted member expression', () => { + expect( + extractCopilotKitIdentifiers('Read `window.copilotKitInternals` at runtime.'), + ).toEqual(['copilotKitInternals']); + expect(extractCopilotKitIdentifiers('Read `CopilotKitApi.copilotKitVersion`.')).toEqual([ + 'CopilotKitApi', + 'copilotKitVersion', + ]); }); it('ignores generic React vocabulary so real answers are not penalized', () => { @@ -55,6 +79,13 @@ describe('extractCopilotKitIdentifiers', () => { expect(ids).toEqual([]); }); + // A subpath import is still a package specifier, not an API surface claim. + it('ignores @copilotkit subpath specifiers', () => { + expect( + extractCopilotKitIdentifiers('Import from `@copilotkit/react-core/copilotKitGhost`.'), + ).toEqual([]); + }); + it('ignores prose and code fences that are not identifiers', () => { const ids = extractCopilotKitIdentifiers( 'The `CopilotKit provider wraps your app` sentence is not an identifier.', @@ -83,6 +114,34 @@ describe('extractCopilotKitIdentifiers', () => { expect( extractCopilotKitIdentifiers('See https://docs.copilotkit.ai/reference/chat for docs.'), ).toEqual([]); + expect( + extractCopilotKitIdentifiers('See www.copilotkit.ai/reference/chat for docs.'), + ).toEqual([]); + }); + + // Bare hostnames are how people actually write links in chat, and the host + // `docs.copilotkit.ai` used to yield the phantom identifier `copilotkit`. + it('never mines identifiers out of a scheme-less hostname', () => { + expect( + extractCopilotKitIdentifiers('See docs.copilotkit.ai/reference/chat for docs.'), + ).toEqual([]); + expect(extractCopilotKitIdentifiers('Docs live on copilotkit.ai these days.')).toEqual([]); + expect(extractCopilotKitIdentifiers('Try the `docs.copilotkit.ai` mirror.')).toEqual([]); + }); + + // Host-stripping is anchored on a real TLD precisely so dotted things that are + // NOT hostnames survive. Anything swallowed here is an identifier we stop + // checking against the sources — the gate would go quiet, not loud. + it('does not mistake a dotted CSS selector chain for a hostname', () => { + expect( + extractCopilotKitIdentifiers('Use `.copilotKitInput.copilotKitInputExpanded` instead.'), + ).toEqual(['copilotKitInput', 'copilotKitInputExpanded']); + }); + + it('does not mistake a version number for a hostname', () => { + expect(extractCopilotKitIdentifiers('On 1.2.3, override `.copilotKitGhost`.')).toEqual([ + 'copilotKitGhost', + ]); }); }); @@ -99,15 +158,18 @@ describe('assessGroundedness', () => { expect(result.reasons).toEqual([]); }); - it('flags a confirmed-bug claim and suppresses the response', () => { + // The decided contract: `suppress` is driven ONLY by identifiers the sources do + // not contain. Claim wording is a penalty, never a gate — a misread of English + // costs 0.35 of confidence, not the reporter's answer. + it('charges a confirmed-bug claim but does not suppress on it', () => { const result = assessGroundedness( '## Bug Confirmed: Cursor Jump in Expanded Mode\n\nThanks for the repro steps!', CHAT_DOCS, ); expect(result.unverifiedClaims).toContain('"bug confirmed"'); - expect(result.penalty).toBeGreaterThan(0); - expect(result.suppress).toBe(true); + expect(result.penalty).toBeCloseTo(0.35, 5); + expect(result.suppress).toBe(false); }); it.each([ @@ -117,47 +179,52 @@ describe('assessGroundedness', () => { ['The fix is to save and restore the selection.', 'asserts the fix'], ['I reproduced this locally on the latest version.', 'claims to have reproduced or tested'], ['We ran the tests and they pass.', 'claims to have reproduced or tested'], - ])('suppresses %j', (response, expectedLabel) => { + ])('charges %j without withholding it', (response, expectedLabel) => { const result = assessGroundedness(response, CHAT_DOCS); expect(result.unverifiedClaims).toContain(expectedLabel); - expect(result.suppress).toBe(true); + expect(result.penalty).toBeGreaterThan(0); + expect(result.suppress).toBe(false); }); - // Suppression is user-visible — the reporter gets the no-answer reply instead - // of a real one — so a false positive costs more than a missed one. These are - // all well-behaved responses of exactly the kind the prompt asks for. - describe('negated claims', () => { - it.each([ - 'This is not a known issue as far as the docs show.', - "I can't determine what the root cause is without reproducing it.", - "I don't know what the fix is — engineering will need to confirm.", - 'I have not reproduced this myself.', - "We haven't tested this against your version.", - "It's unclear whether this is a real bug or expected behavior.", - 'No bug confirmed here — the docs describe this as intended.', - ])('does not suppress %j', (response) => { - const result = assessGroundedness(response, CHAT_DOCS); - expect(result.unverifiedClaims).toEqual([]); - expect(result.suppress).toBe(false); + // The reported basis has to equal what was actually billed, or the log line + // understates the deduction it is supposed to explain. + describe('reported reasons match what was charged', () => { + it('charges one claim for a sentence that trips several patterns', () => { + const result = assessGroundedness('This is a known bug.', CHAT_DOCS); + + expect(result.unverifiedClaims).toHaveLength(1); + expect(result.penalty).toBeCloseTo(0.35, 5); + expect(result.reasons).toEqual([ + `unverifiable claims: ${result.unverifiedClaims.join(', ')}`, + ]); }); - it('still catches an assertion in a later sentence', () => { - // The negation belongs to the first sentence only. + it('charges one claim for the same wording repeated across sentences', () => { const result = assessGroundedness( - 'I have not reproduced this. Root cause is a re-render on every keystroke.', + 'Root cause is a re-render. Root cause is a layout thrash.', CHAT_DOCS, ); - expect(result.unverifiedClaims).toContain('asserts a root cause'); - expect(result.suppress).toBe(true); + + expect(result.unverifiedClaims).toHaveLength(1); + expect(result.penalty).toBeCloseTo(0.35, 5); }); - it('still catches an assertive occurrence when another is negated', () => { + it('reports every distinct claim it charged, and charges every claim it reports', () => { const result = assessGroundedness( - "It's unclear whether the root cause is the layout. Bug confirmed regardless.", + 'Bug confirmed. Root cause is a re-render. The fix is trivial.', CHAT_DOCS, ); - expect(result.unverifiedClaims).toContain('"bug confirmed"'); - expect(result.suppress).toBe(true); + + // Three distinct accusations → three reported labels, and the raw + // deduction is the count times the rate (clipped by the ceiling here). + expect(result.unverifiedClaims).toHaveLength(3); + expect(result.unverifiedClaims.length * 0.35).toBeGreaterThan(MAX_GROUNDEDNESS_PENALTY); + expect(result.penalty).toBe(MAX_GROUNDEDNESS_PENALTY); + expect(result.reasons[0]).toBe( + `unverifiable claims: ${result.unverifiedClaims.join(', ')}`, + ); + // Suppression is unaffected: three claims, no invented identifier. + expect(result.suppress).toBe(false); }); }); @@ -256,7 +323,8 @@ describe('assessGroundedness', () => { expect(() => assessGroundedness('Anything at all.', partial)).not.toThrow(); }); - // The exact response from CopilotKit/CopilotKit#6167, condensed. + // The exact response from CopilotKit/CopilotKit#6167, condensed. It is withheld + // on its two invented class names alone — the claim wording only adds penalty. it('would have caught the #6167 response', () => { const response = [ '## Bug Confirmed: Cursor Jump in Expanded Mode', @@ -272,25 +340,46 @@ describe('assessGroundedness', () => { const result = assessGroundedness(response, CHAT_DOCS); - expect(result.suppress).toBe(true); - expect(result.unverifiedClaims.length).toBeGreaterThanOrEqual(2); expect(result.unsourcedIdentifiers).toEqual([ 'copilotKitInputControls', 'copilotKitInputControlsExpanded', ]); + expect(result.unsourcedIdentifiers.length).toBeGreaterThanOrEqual( + SUPPRESS_AT_UNSOURCED_IDENTIFIERS, + ); + expect(result.suppress).toBe(true); + expect(result.unverifiedClaims.length).toBeGreaterThanOrEqual(2); expect(result.penalty).toBe(MAX_GROUNDEDNESS_PENALTY); }); + + // Strip the fabricated class names out of #6167 and the same prose is published + // with a penalty. That is the decided trade: the claim wording never withholds. + it('does not withhold the #6167 prose once the invented class names are gone', () => { + const result = assessGroundedness( + [ + '## Bug Confirmed: Cursor Jump in Expanded Mode', + 'This is a real bug worth fixing in the core.', + ].join('\n'), + CHAT_DOCS, + ); + + expect(result.suppress).toBe(false); + expect(result.penalty).toBeGreaterThan(0); + }); }); /** * Corpus organized by RESPONSE SHAPE, not by regex. * - * The unit under test is "given a response that looks like THIS, do we post it?" — - * so each row names a shape a real answer takes (bare assertion, negated assertion, - * hedged assertion, markdown bullets, a cited docs URL, a code fence) and pins the - * publish/withhold decision for it. Rows are shape-complete rather than - * pattern-complete on purpose: it is the shapes that regress when the matching - * internals get rewritten. + * Two outcomes are now independent and both are pinned per row: + * + * - `suppress` — does the reporter see this answer? Driven ONLY by identifiers the + * sources do not contain, which is checkable against those sources. + * - `claimCharged` — did the claim-phrase penalty fire? Driven by English wording, + * which is fallible, so its only consequence is a lower confidence score. + * + * Every negation shape that previously misbehaved has a row here, and they all + * assert `suppress: false` — no wording, negated or not, can withhold a response. */ interface CorpusRow { shape: string; @@ -298,6 +387,8 @@ interface CorpusRow { sources?: SearchResult[]; /** The decision that matters: does the reporter see this answer? */ suppress: boolean; + /** Independent of `suppress`: was the claim-phrase penalty billed? */ + claimCharged: boolean; /** Pinned only where the arithmetic is the point of the row. */ penalty?: number; unsourcedIdentifiers?: string[]; @@ -323,57 +414,92 @@ const CORPUS: CorpusRow[] = [ { shape: 'bare assertion of a root cause', response: 'Root cause is a re-render on every keystroke.', - suppress: true, + suppress: false, + claimCharged: true, penalty: 0.35, }, + // ---- negation shapes: all publishable, all still charged ------------------- { - shape: 'assertion cancelled by a negation BEFORE it', + shape: 'LEADING negation ("I cannot tell what the root cause is")', response: 'I cannot tell what the root cause is from the docs alone.', suppress: false, - penalty: 0, + claimCharged: true, + penalty: 0.35, }, { - shape: 'assertion cancelled by a negation AFTER it in the same sentence', + shape: 'TRAILING negation ("the root cause is not obvious")', response: 'The root cause is not obvious from the docs.', suppress: false, - penalty: 0, + claimCharged: true, + penalty: 0.35, }, { - // Version numbers must not fragment the sentence, or the negation lands in - // a different fragment than the claim and a good answer gets withheld. - shape: 'negated assertion whose sentence contains a version number', - response: "I can't reproduce on 1.2.3, so the root cause is a mystery.", + shape: 'INCIDENTAL in-clause negation, real assertion in the next clause', + response: 'I cannot reproduce it, but the root cause is a re-render.', suppress: false, - penalty: 0, + claimCharged: true, + penalty: 0.35, }, { - shape: 'assertion cancelled by a trailing uncertainty marker', - response: 'What the root cause is remains unclear.', + shape: 'HEDGE-THEN-ASSERT ("Bug confirmed, though I have no repro steps")', + response: 'Bug confirmed, though I have no repro steps.', suppress: false, - penalty: 0, + claimCharged: true, + penalty: 0.35, }, { - // Hedging does not buy the right to assert. The hedge-density penalty is a - // separate, softer signal; it must not double as an assertion escape hatch. - shape: 'hedged assertion ("possibly ...")', - response: 'Possibly the root cause is a re-render.', - suppress: true, + shape: 'assertion with a trailing "no workaround" clause', + response: 'This is a known issue with no workaround.', + suppress: false, + claimCharged: true, penalty: 0.35, }, { - shape: 'assertion in the sentence AFTER a negated one', + shape: '"no doubt" — a negator that is really an intensifier', + response: 'There is no doubt this is a real bug.', + suppress: false, + claimCharged: true, + penalty: 0.35, + }, + { + // `no-cache` contains a `no` that is not a word of English negation at all. + shape: '"no-cache" substring inside an unrelated technical token', + response: 'Root cause is the `no-cache` header on the docs route.', + suppress: false, + claimCharged: true, + penalty: 0.35, + }, + { + shape: 'sentence-crossing negation, assertion in the following sentence', response: 'I have not reproduced this. Root cause is a re-render.', - suppress: true, + suppress: false, + claimCharged: true, penalty: 0.35, }, { - // A newline ends a thought as firmly as a period does; a negation on the - // previous bullet says nothing about this one. shape: 'negation and assertion on separate markdown bullets', response: '- No workaround exists yet\n- Root cause is a re-render on every keystroke', - suppress: true, + suppress: false, + claimCharged: true, penalty: 0.35, }, + { + // No claim pattern matches at all here — nothing to charge, nothing to gate. + shape: 'honest non-answer that names no claim wording', + response: 'I have not reproduced this myself, so engineering should take a look.', + suppress: false, + claimCharged: false, + penalty: 0, + }, + { + // Hedging does not buy the right to assert, and it does not excuse it either. + shape: 'hedged assertion ("possibly ...")', + response: 'Possibly the root cause is a re-render.', + suppress: false, + claimCharged: true, + penalty: 0.35, + }, + // ---- identifier shapes: the only thing that withholds ---------------------- { shape: 'cites a docs URL whose path documents the identifier it names', response: @@ -381,17 +507,30 @@ const CORPUS: CorpusRow[] = [ 'https://docs.copilotkit.ai/reference/components/CopilotKitProvider.', sources: [urlSource('https://docs.copilotkit.ai/reference/components/CopilotKitProvider')], suppress: false, + claimCharged: false, penalty: 0, unsourcedIdentifiers: [], }, { // The prompt asks for docs links, so the hostname appears in most good // answers. It must never register as an identifier of its own. - shape: 'cites a docs URL absent from the sources', + shape: 'cites a docs URL (with scheme) absent from the sources', response: 'Full details live at https://docs.copilotkit.ai/reference/components/chat/CopilotChat.', sources: NO_URL_DOCS, suppress: false, + claimCharged: false, + penalty: 0, + unsourcedIdentifiers: [], + }, + { + // Same link written the way people actually type it. `docs.copilotkit.ai` + // used to yield the phantom identifier `copilotkit`. + shape: 'cites a docs URL (SCHEME-LESS host) absent from the sources', + response: 'Full details live at docs.copilotkit.ai/reference/components/chat/CopilotChat.', + sources: NO_URL_DOCS, + suppress: false, + claimCharged: false, penalty: 0, unsourcedIdentifiers: [], }, @@ -402,6 +541,7 @@ const CORPUS: CorpusRow[] = [ 'See https://docs.copilotkit.ai/reference/copilotKitPhantomHook for ' + '`copilotKitPhantomHook`.', suppress: false, + claimCharged: false, penalty: 0.15, unsourcedIdentifiers: ['copilotKitPhantomHook'], }, @@ -416,33 +556,42 @@ const CORPUS: CorpusRow[] = [ '```', ].join('\n'), suppress: true, + claimCharged: false, unsourcedIdentifiers: ['copilotKitGhostA', 'copilotKitGhostB'], }, { - shape: 'one invented identifier written in two casings', + shape: 'one invented identifier written in two casings counts once', response: 'Override `.copilotKitFoo` and `.CopilotKitFoo` to fix it.', suppress: false, + claimCharged: false, penalty: 0.15, unsourcedIdentifiers: ['copilotKitFoo'], }, { - // Two claim patterns fire on this one sentence; it is still one claim. - shape: '"known bug" assertion matching several patterns at once', - response: 'This is a known bug.', + shape: 'two invented identifiers written as a call and as JSX', + response: 'Call `useCopilotKitGhost()` inside ``.', suppress: true, - penalty: 0.35, + claimCharged: false, + unsourcedIdentifiers: ['useCopilotKitGhost', 'CopilotKitGhostPanel'], }, { - shape: '"no doubt" used as an intensifier, not a negation', - response: 'There is no doubt this is a real bug.', - suppress: true, + // Two claim patterns fire on this one sentence; it is still one claim. + shape: '"known bug" assertion matching several patterns at once', + response: 'This is a known bug.', + suppress: false, + claimCharged: true, penalty: 0.35, }, { - shape: '"no question about it" used as an intensifier', - response: 'No question about it, bug confirmed.', + // The #6167 shape: unsupportable prose AND invented names. The names gate it. + shape: 'claim wording plus two invented identifiers', + response: + 'Bug confirmed. Override `.copilotKitGhostA` and `.copilotKitGhostB` to work around it.', suppress: true, - penalty: 0.35, + claimCharged: true, + // 0.35 claim + 2 × 0.15 identifiers = 0.65, clipped to the ceiling. + penalty: MAX_GROUNDEDNESS_PENALTY, + unsourcedIdentifiers: ['copilotKitGhostA', 'copilotKitGhostB'], }, { shape: 'grounded answer that asserts nothing it cannot support', @@ -450,23 +599,46 @@ const CORPUS: CorpusRow[] = [ 'You can replace the chat input with the `input` prop on the `CopilotChat` ' + 'component. That keeps your own state, so you control the cursor.', suppress: false, + claimCharged: false, penalty: 0, unsourcedIdentifiers: [], }, ]; describe('assessGroundedness response-shape corpus', () => { - it.each(CORPUS)('$shape', ({ response, sources, suppress, penalty, unsourcedIdentifiers }) => { - const result = assessGroundedness(response, sources ?? CHAT_DOCS); - - expect(result.suppress).toBe(suppress); - if (penalty !== undefined) expect(result.penalty).toBeCloseTo(penalty, 5); - if (unsourcedIdentifiers !== undefined) { - expect(result.unsourcedIdentifiers).toEqual(unsourcedIdentifiers); + it.each(CORPUS)( + '$shape', + ({ response, sources, suppress, claimCharged, penalty, unsourcedIdentifiers }) => { + const result = assessGroundedness(response, sources ?? CHAT_DOCS); + + expect(result.suppress).toBe(suppress); + expect(result.unverifiedClaims.length > 0).toBe(claimCharged); + if (penalty !== undefined) expect(result.penalty).toBeCloseTo(penalty, 5); + if (unsourcedIdentifiers !== undefined) { + expect(result.unsourcedIdentifiers).toEqual(unsourcedIdentifiers); + } + }, + ); + + it('never lets claim wording alone withhold a response', () => { + const withClaimsOnly = CORPUS.filter((row) => row.claimCharged); + expect(withClaimsOnly.length).toBeGreaterThan(0); + + for (const row of withClaimsOnly) { + const result = assessGroundedness(row.response, row.sources ?? CHAT_DOCS); + if (result.unsourcedIdentifiers.length < SUPPRESS_AT_UNSOURCED_IDENTIFIERS) { + expect(result.suppress).toBe(false); + } } }); it('pins the suppression bar to the exported threshold', () => { + const oneInvented = assessGroundedness('Override `.copilotKitGhostA`.', CHAT_DOCS); + expect(oneInvented.unsourcedIdentifiers).toHaveLength( + SUPPRESS_AT_UNSOURCED_IDENTIFIERS - 1, + ); + expect(oneInvented.suppress).toBe(false); + const twoInvented = assessGroundedness( 'Override `.copilotKitGhostA` and `.copilotKitGhostB`.', CHAT_DOCS, diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts index 5811e68b..c7c3cd87 100644 --- a/packages/outpost/ai/src/groundedness.ts +++ b/packages/outpost/ai/src/groundedness.ts @@ -16,14 +16,36 @@ import type { SearchResult } from './types.js'; * cheap, deterministic, and unit-testable — the prompt rules in generator.ts are * the request, this is the enforcement. * - * Shape of the analysis: **normalize, then split, then judge each sentence.** - * Earlier revisions matched claim patterns against the whole response and then - * looked backwards over a fixed character window for a negation. That failed in - * both directions — it missed trailing negations ("the root cause is not - * obvious"), let a negation on one markdown bullet cancel an assertion on the - * next, and could slice `cannot` into a bare `not`. Sentences are the unit a - * negation actually scopes over, so we cut the text into them first and ask each - * one, independently, "is this asserted here?". + * ## Two signals, two very different consequences + * + * The module produces two outcomes and they are deliberately wired to different + * levers, because they are not equally trustworthy: + * + * 1. **`suppress` — the objective signal only.** A response is withheld from the + * public thread when it names CopilotKit identifiers that appear in *none* of + * the retrieved sources (`unsourcedIdentifiers` ≥ + * `SUPPRESS_AT_UNSOURCED_IDENTIFIERS`). That is a checkable fact: the name is + * either in the sources we handed the model or it is not. No English is parsed + * to reach it. + * + * 2. **Claim phrases — penalty only.** "Bug confirmed", "root cause is", "I + * reproduced this" are still detected, still charged + * `PENALTY_PER_UNVERIFIED_CLAIM`, and still reported on `unverifiedClaims` and + * `reasons`. They no longer contribute to `suppress` at all. + * + * Why: deciding whether a sentence *asserts* a claim or *denies* it is natural + * language negation, and three successive regex attempts at it failed in three + * different ways — a backwards character window suppressed "the root cause is not + * obvious", and the sentence-scoped negator list waved through "This is a known + * issue with no workaround", "I cannot reproduce it, but the root cause is a + * re-render", and "Bug confirmed, though I have no repro steps". Negation is not + * regex-tractable, so it must not gate a user-visible publish/withhold decision. + * The negation machinery is gone entirely rather than tuned again. + * + * What that costs and buys: a misread claim phrase now deducts 0.35 of confidence + * instead of withholding the reporter's answer. The penalty still pulls the score + * under the escalation gate, so a human is pulled in and the disclaimer still + * lands — the consequence of a misread is a lower score, never a lost reply. */ /** The kind of unsupportable claim a pattern detects. Several patterns can share one. */ @@ -36,6 +58,10 @@ type ClaimCategory = 'confirmation' | 'bug-validity' | 'root-cause' | 'fix' | 'r * bug" trips both the real-bug pattern and the known-bug pattern; it is still one * claim, and billing it twice made a single sentence cost more than two distinct * fabrications. + * + * Order matters: the FIRST pattern to match a category supplies the label that + * gets reported, so the more specific wording is listed first ("known issue" + * should read as "claims a known bug", not as the generic real-bug assertion). */ const UNVERIFIED_CLAIM_PATTERNS: Array<{ pattern: RegExp; @@ -49,14 +75,14 @@ const UNVERIFIED_CLAIM_PATTERNS: Array<{ category: 'confirmation', }, { - pattern: - /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, - label: 'asserts the report is a real bug', + pattern: /\bknown\s+(?:bug|issue|regression)\b/i, + label: 'claims a known bug', category: 'bug-validity', }, { - pattern: /\bknown\s+(?:bug|issue|regression)\b/i, - label: 'claims a known bug', + pattern: + /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, + label: 'asserts the report is a real bug', category: 'bug-validity', }, { @@ -74,57 +100,28 @@ const UNVERIFIED_CLAIM_PATTERNS: Array<{ ]; /** - * Words that genuinely reverse a claim inside its own sentence. - * - * The patterns above match assertions, but the same words appear in exactly the - * responses the prompt asks for: "this is **not** a known issue", "I **can't** - * determine what the root cause is without reproducing it", "I **don't** know what - * the fix is". Suppression is user-visible (the reporter gets the no-answer reply - * instead of a real one), so a false positive costs more than a missed one. + * Links are replaced with a placeholder before any analysis. * - * Hedges — `maybe`, `possibly`, `suspect`, `guess` — are deliberately NOT here. - * They do not reverse a claim, they only soften its delivery, and "possibly the - * root cause is X" is precisely the confident-guess shape this gate exists to - * catch. Hedging is priced separately by the hedge-density penalty below. + * The prompt actively instructs the bot to cite docs URLs, so + * `https://docs.copilotkit.ai/...` shows up in most *good* answers, and reading + * the hostname as a declaration of `.copilotkit` invented a fabrication out of a + * correct citation. Since `suppress` now rides entirely on the identifier signal, + * a phantom identifier is no longer merely a wrong penalty — two of them withhold + * a correct answer. * - * `no` stays, because "no bug confirmed here" is a real negation — but see - * INTENSIFIER_PATTERN for the phrases where it means the opposite. - */ -const NEGATOR_PATTERN = - /\b(?:not|never|cannot|can\s+not|unable|without|unclear|unsure|unconfirmed|unknown|undetermined|no|nor|none|neither)\b|n['’]t\b/i; - -/** - * Phrases where a negator is actually an intensifier: "there is **no doubt** this - * is a real bug" asserts harder than the plain sentence does. Neutralized before - * the negator scan so they cannot wave a claim through. - */ -const INTENSIFIER_PATTERN = /\b(?:no|without|beyond)\s+(?:a\s+)?(?:doubt|question)s?\b/gi; -const INTENSIFIER_REPLACEMENT = 'certainly'; - -/** - * URLs are replaced with this before any analysis. + * Two forms are stripped: fully-qualified links (`https://…`, `www.…`) and the + * scheme-less host form people actually type in chat and issue comments + * (`docs.copilotkit.ai/reference`). The host form is anchored on a known TLD so a + * version number (`1.2.3`) or a CSS selector (`.copilotKitInput`) is never + * mistaken for a hostname — stripping those would hide real identifiers. * - * Two reasons. Identifiers: the prompt actively instructs the bot to cite docs - * URLs, so `https://docs.copilotkit.ai/...` shows up in most *good* answers, and - * reading the hostname as a declaration of `.copilotkit` invented a fabrication - * out of a correct citation. Sentences: a URL is full of `.` and `?`, which would - * shred one sentence into several. The placeholder carries no `.`, so it is inert - * on both counts. + * The placeholder carries no `.`, so nothing downstream can mine it. */ const URL_PATTERN = /\b(?:https?:\/\/|www\.)[^\s<>()[\]{}"'`]+/gi; +const BARE_HOST_PATTERN = + /\b(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+(?:ai|app|co|com|dev|io|net|org|sh)\b(?:\/[^\s<>()[\]{}"'`]*)?/gi; const URL_PLACEHOLDER = ' [url] '; -/** - * Sentence boundary: terminal punctuation OR a line break. A newline ends a - * thought as firmly as a period does — markdown answers are mostly bullets, and a - * negation on one bullet says nothing about the next. - * - * A `.` between digits is not a boundary, so `v1.2.3` stays inside its sentence. - * Splitting a version number would strand the negation ("I can't reproduce this on - * 1.2.3") in a different fragment from the claim, which suppresses a good answer. - */ -const SENTENCE_BOUNDARY = /(?`, ``, `` — JSX is how a component name is usually written. */ +const JSX_WRAPPER = /^<\/?\s*([A-Za-z_$][\w$.-]*)\s*\/?>$/; +/** `foo()`, `foo({ debug: true })` — a call is still a claim about an API surface. */ +const CALL_EXPRESSION = /^([^()]*?)\(\s*[^()]*\)$/; +/** A bare identifier, optionally selector-prefixed and optionally dotted. */ +const IDENTIFIER_PATH = /^[.#]?[A-Za-z_$][\w$-]*(?:\.[A-Za-z_$][\w$-]*)*$/; + /** Hedges allowed before the density penalty starts. */ const HEDGE_FREE_ALLOWANCE = 2; @@ -169,44 +173,72 @@ export const MAX_GROUNDEDNESS_PENALTY = 0.6; * publish. One could be a formatting artifact; two is a pattern of fabrication * (#6167 shipped exactly two). * - * Exported so tests pin the threshold by name instead of hard-coding the number. + * This is the ONLY input to `suppress`, so the extraction rules below carry the + * whole gate. Exported so tests pin the threshold by name instead of hard-coding + * the number. */ export const SUPPRESS_AT_UNSOURCED_IDENTIFIERS = 2; export interface GroundednessAssessment { /** Amount to deduct from the confidence score (0 – MAX_GROUNDEDNESS_PENALTY). */ penalty: number; - /** Verification claims the bot is not entitled to make. */ + /** + * Verification claims the bot is not entitled to make. One entry per claim + * actually charged, so the reported basis equals the deduction. Penalty-only: + * these never set `suppress`. + */ unverifiedClaims: string[]; /** CopilotKit identifiers named in the response but absent from every source. */ unsourcedIdentifiers: string[]; /** Total hedge markers found. */ hedgeCount: number; /** - * True when the response makes a claim we cannot stand behind. The caller is + * True when the response names identifiers no retrieved source contains — the + * one signal here that is verifiable against those sources. The caller is * expected to withhold it from the public thread and escalate to a human - * instead — a lowered score alone does not stop a post. + * instead; a lowered score alone does not stop a post. */ suppress: boolean; /** Human-readable reasons, for logs and the dashboard. */ reasons: string[]; } -/** Blank out URLs so neither identifier extraction nor sentence splitting sees them. */ +/** Blank out links so identifier extraction never reads a hostname as an API name. */ function stripUrls(text: string): string { - return text.replace(URL_PATTERN, URL_PLACEHOLDER); + return text.replace(URL_PATTERN, URL_PLACEHOLDER).replace(BARE_HOST_PATTERN, URL_PLACEHOLDER); } /** - * Cut text into the units a negation scopes over. Terminal punctuation and line - * breaks both end a sentence, so a negated markdown bullet cannot reach the next - * bullet's assertion. + * Reduce a backticked token to the CopilotKit-named identifiers it declares. + * + * Widened past the bare-name guard it started with, because the identifier signal + * is now the whole gate: `useCopilotKitFoo()`, `` and + * `window.copilotKitFoo` are the same claim as `copilotKitFoo`, and letting a + * fabrication through on syntax alone would defeat the check. Package specifiers + * (`@copilotkit/react-core`) stay excluded — they are stable public knowledge, not + * a claim about the retrieved page. + * + * Anything that isn't identifier-shaped after unwrapping (prose, a fenced snippet, + * a path) yields nothing. */ -function splitSentences(text: string): string[] { - return text - .split(SENTENCE_BOUNDARY) - .map((sentence) => sentence.trim()) - .filter((sentence) => sentence.length > 0); +function identifierSegments(rawToken: string): string[] { + let token = rawToken.trim(); + if (!token) return []; + // Package specifier, including subpath imports. + if (token.startsWith('@')) return []; + if (!/copilotkit/i.test(token)) return []; + + const jsx = JSX_WRAPPER.exec(token); + if (jsx) token = jsx[1]; + + const call = CALL_EXPRESSION.exec(token); + if (call) token = call[1].trim(); + + if (!IDENTIFIER_PATH.test(token)) return []; + + // A dotted form names a member; report the segments that carry our name so the + // grounding lookup compares something a source could plausibly contain. + return token.split('.').filter((segment) => segment && /copilotkit/i.test(segment)); } /** @@ -215,31 +247,34 @@ function splitSentences(text: string): string[] { * Exported for testing: the extraction rules are the part most likely to drift * into false positives, so they're pinned directly. * - * Dedup is case-folded to match the case-insensitive grounding comparison in - * `assessGroundedness` — otherwise `.copilotKitFoo` plus `.CopilotKitFoo` counts - * as two fabrications and clears the suppression bar by itself. The first - * spelling seen is the one reported, so log lines quote the response. + * Results are in the order the names appear in the response, so logs read like the + * answer. Dedup is case-folded to match the case-insensitive grounding comparison + * in `assessGroundedness` — otherwise `.copilotKitFoo` plus `.CopilotKitFoo` counts + * as two fabrications and clears the suppression bar by itself. The first spelling + * seen is the one reported. */ export function extractCopilotKitIdentifiers(response: string): string[] { - const found = new Map(); - const remember = (identifier: string): void => { - const key = identifier.toLowerCase(); - if (!found.has(key)) found.set(key, identifier); - }; - const text = stripUrls(response); + const hits: Array<{ at: number; name: string }> = []; for (const match of text.matchAll(CSS_CLASS_PATTERN)) { - remember(match[1]); + hits.push({ at: match.index ?? 0, name: match[1] }); } for (const match of text.matchAll(BACKTICKED_PATTERN)) { - const token = match[1].trim(); - // Skip package specifiers and anything that isn't a bare identifier. - if (token.startsWith('@')) continue; - if (!/copilotkit/i.test(token)) continue; - if (!/^[.#]?[A-Za-z_$][\w$-]*$/.test(token)) continue; - remember(token.replace(/^[.#]/, '')); + const token = match[1]; + const tokenStart = (match.index ?? 0) + 1; + for (const segment of identifierSegments(token)) { + hits.push({ at: tokenStart + token.indexOf(segment), name: segment }); + } + } + + hits.sort((a, b) => a.at - b.at); + + const found = new Map(); + for (const { name } of hits) { + const key = name.toLowerCase(); + if (!found.has(key)) found.set(key, name); } return [...found.values()]; @@ -265,33 +300,25 @@ export function assessGroundedness( if (!response) return empty; - const normalized = stripUrls(response).replace(INTENSIFIER_PATTERN, INTENSIFIER_REPLACEMENT); + const normalized = stripUrls(response); - // A claim counts where it is asserted, sentence by sentence: a sentence with no - // genuine negator anywhere in it — before OR after the matched wording — - // asserts what it says. One negated mention therefore does not excuse an - // assertive one elsewhere, and an assertion is not excused by a negation that - // belongs to a neighbouring sentence. + // One charge per claim CATEGORY over the whole response, and the reported + // labels ARE the charged ones — `unverifiedClaims.length` is the multiplier, so + // the log line can never understate the deduction it explains. Overlapping + // wordings of one accusation ("this is a known bug") cost one claim, not two. // - // Charging is once per sentence per claim CATEGORY, so overlapping wordings of - // one accusation ("this is a known bug") cost one claim, not two. + // No negation analysis: the response is scanned as written. A denial that + // happens to contain the wording is charged too, and that is the accepted + // trade — see the module comment. The consequence is a lower score, never a + // withheld reply. const unverifiedClaims: string[] = []; - const seenLabels = new Set(); - let chargeableClaims = 0; - - for (const sentence of splitSentences(normalized)) { - if (NEGATOR_PATTERN.test(sentence)) continue; - - const categories = new Set(); - for (const { pattern, label, category } of UNVERIFIED_CLAIM_PATTERNS) { - if (!pattern.test(sentence)) continue; - categories.add(category); - if (!seenLabels.has(label)) { - seenLabels.add(label); - unverifiedClaims.push(label); - } - } - chargeableClaims += categories.size; + const chargedCategories = new Set(); + + for (const { pattern, label, category } of UNVERIFIED_CLAIM_PATTERNS) { + if (chargedCategories.has(category)) continue; + if (!pattern.test(normalized)) continue; + chargedCategories.add(category); + unverifiedClaims.push(label); } // Sources are searched as one haystack: an identifier documented on any @@ -320,15 +347,14 @@ export function assessGroundedness( const excessHedges = Math.max(0, hedgeCount - HEDGE_FREE_ALLOWANCE); const penalty = Math.min( - chargeableClaims * PENALTY_PER_UNVERIFIED_CLAIM + + unverifiedClaims.length * PENALTY_PER_UNVERIFIED_CLAIM + unsourcedIdentifiers.length * PENALTY_PER_UNSOURCED_IDENTIFIER + excessHedges * PENALTY_PER_EXCESS_HEDGE, MAX_GROUNDEDNESS_PENALTY, ); - const suppress = - unverifiedClaims.length > 0 || - unsourcedIdentifiers.length >= SUPPRESS_AT_UNSOURCED_IDENTIFIERS; + // Objective signal only. Claim wording is priced above and stops there. + const suppress = unsourcedIdentifiers.length >= SUPPRESS_AT_UNSOURCED_IDENTIFIERS; const reasons: string[] = []; if (unverifiedClaims.length) { diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index e0881a02..3ffce467 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -210,7 +210,9 @@ describe('AIPipeline', () => { 'never hedges about completeness at score %s', async (score) => { const text = await disclaimerFor(score); - expect(text).not.toMatch(/may be incomplete|might be incomplete|may not be accurate/i); + expect(text).not.toMatch( + /may be incomplete|might be incomplete|may not be accurate/i, + ); expect(text).toContain('This is an AI-generated response.'); }, ); @@ -242,7 +244,10 @@ describe('AIPipeline', () => { ]); }); - it('marks a response that confirms a bug as suppressed', async () => { + // Withholding is driven by the objective signal only — identifiers no + // retrieved source contains. Claim wording is fallible English, so it + // buys a penalty and an escalation, never a withheld reply. + it('penalizes a bug-confirming response into escalation without withholding it', async () => { mockGenerate.mockResolvedValue({ ...sampleGeneratedResponse, text: '## Bug Confirmed: Cursor Jump\n\nRoot cause is a re-render.', @@ -250,6 +255,25 @@ describe('AIPipeline', () => { const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + expect(result.groundedness.unverifiedClaims.length).toBeGreaterThan(0); + expect(result.suppressed).toBe(false); + expect(result.groundedness.suppress).toBe(false); + // The reporter still gets a human: the penalty clears the gate on its own. + expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); + }); + + it('marks a response naming identifiers absent from the sources as suppressed', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Override `.copilotKitGhostA` and `.copilotKitGhostB` to fix it.', + }); + + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.groundedness.unsourcedIdentifiers).toEqual([ + 'copilotKitGhostA', + 'copilotKitGhostB', + ]); expect(result.suppressed).toBe(true); expect(result.groundedness.suppress).toBe(true); expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); @@ -268,6 +292,28 @@ describe('AIPipeline', () => { confidenceCalibration: 0.15, }); + // 0.85 + 0.15 calibration = 1.0, minus the capped 0.6 penalty = 0.4. + // The boost cannot outrun the deduction, and the answer lands on the + // gate rather than above it. + expect(withBoost.groundedness.penalty).toBeCloseTo(0.6, 5); + expect(withBoost.confidenceScore).toBeCloseTo(0.4, 5); + expect(withBoost.confidenceScore).toBeLessThanOrEqual(AI_CONFIDENCE.ESCALATE); + expect(withBoost.confidenceLevel).toBe(ConfidenceLevel.LOW); + }); + + // A boost cannot lift a withheld answer over the escalation gate either: + // the suppression clamp sits below it by construction. + it('keeps a suppressed response below the gate even with a positive boost', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Override `.copilotKitGhostA` and `.copilotKitGhostB` to fix it.', + }); + + const withBoost = await pipeline.generateSupportResponse('q', { + source: 'github', + confidenceCalibration: 0.15, + }); + expect(withBoost.suppressed).toBe(true); expect(withBoost.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); }); @@ -474,7 +520,11 @@ describe('AIPipeline confidence calibration', () => { pipeline = createPipeline(); mockSearchDocs.mockResolvedValue(sampleSearchResults); mockGenerate.mockResolvedValue(sampleGeneratedResponse); // generator score 0.85 - mockScore.mockResolvedValue({ ...sampleConfidence, score: 0.6, level: ConfidenceLevel.MEDIUM }); + mockScore.mockResolvedValue({ + ...sampleConfidence, + score: 0.6, + level: ConfidenceLevel.MEDIUM, + }); mockFormat.mockReturnValue({ text: 'Formatted response', truncated: false }); }); @@ -493,7 +543,11 @@ describe('AIPipeline confidence calibration', () => { }); it('clamps the calibrated score to at most 1', async () => { - mockScore.mockResolvedValue({ ...sampleConfidence, score: 0.95, level: ConfidenceLevel.HIGH }); + mockScore.mockResolvedValue({ + ...sampleConfidence, + score: 0.95, + level: ConfidenceLevel.HIGH, + }); const result = await pipeline.generateSupportResponse('q', { source: 'discord', confidenceCalibration: 0.2, // min(0.85, 0.95)=0.85 + 0.2 = 1.05 → clamp @@ -502,7 +556,11 @@ describe('AIPipeline confidence calibration', () => { }); it('clamps the calibrated score to at least 0', async () => { - mockScore.mockResolvedValue({ ...sampleConfidence, score: 0.05, level: ConfidenceLevel.LOW }); + mockScore.mockResolvedValue({ + ...sampleConfidence, + score: 0.05, + level: ConfidenceLevel.LOW, + }); const result = await pipeline.generateSupportResponse('q', { source: 'discord', confidenceCalibration: -0.2, // min(0.85, 0.05)=0.05 - 0.2 = -0.15 → clamp diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index 15bb4516..8952f9df 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -148,19 +148,30 @@ const highConfidenceResult = { suppressed: false, }; -/** A response the groundedness check refuses to publish (see #6167). */ +/** + * A response the groundedness check refuses to publish (see #6167). + * + * Withholding is driven solely by identifiers no retrieved source contains, and + * the bar is two — so this fixture carries the two invented class names #6167 + * shipped. The claim wording rides along as penalty only; it does not withhold. + */ const suppressedResult = { ...highConfidenceResult, - response: '## Bug Confirmed: Cursor Jump\n\nOverride `.copilotKitInputControls`.', + response: + '## Bug Confirmed: Cursor Jump\n\nOverride `.copilotKitInputControls` and ' + + '`.copilotKitInputControlsExpanded`.', confidenceLevel: 'LOW', confidenceScore: 0.32, groundedness: { - penalty: 0.5, + penalty: 0.6, unverifiedClaims: ['"bug confirmed"'], - unsourcedIdentifiers: ['copilotKitInputControls'], + unsourcedIdentifiers: ['copilotKitInputControls', 'copilotKitInputControlsExpanded'], hedgeCount: 0, suppress: true, - reasons: ['unverifiable claims: "bug confirmed"'], + reasons: [ + 'unverifiable claims: "bug confirmed"', + 'identifiers absent from sources: copilotKitInputControls, copilotKitInputControlsExpanded', + ], }, suppressed: true, }; From db6c691ebc35d9a4124a9f096dddc292d133ec7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:51:42 -0400 Subject: [PATCH 39/83] fix(ai): enforce the groundedness gate at the pipeline boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppression was enforced at ONE consumer (the queue handler), so every other consumer published ungrounded text. Reviewers found three bypasses: 1. apps/web/src/app/api/qa/route.ts streamed `result.response` to the browser without ever reading `suppressed`. 2. AIPipeline.generateStreamingResponse → ResponseGenerator.generateStream produced no assessment at all — no groundedness, no penalty, no suppression, no disclaimer. The gate did not exist on that path. 3. The queue handler checked `pipelineResult.suppressed` BEFORE its SHADOW_MODE branch, so in shadow mode a suppressed response wrote no shadow record — the responses most worth studying stopped being logged. Fixed at the boundary instead of at each consumer: the pipeline itself now withholds the ungrounded text, so every consumer inherits the gate. - `generateSupportResponse` swaps the new exported SUPPRESSED_RESPONSE_TEXT into `formatted` when `groundedness.suppress` is true. `response` still carries the ORIGINAL draft — a human picking up the escalation works from it — and `suppressed`/`groundedness` stay on the result for analytics. Publishing `formatted` is now safe by construction on every platform target. - The replacement copy already promises a human follow-up, so it is paired with the plain AI_DISCLAIMER rather than stacking AI_DISCLAIMER_ESCALATED on top. - `generateStreamingResponse` now buffers the model stream, assesses the complete text, then yields — either the original chunk boundaries or, when suppressed, only SUPPRESSED_RESPONSE_TEXT. Groundedness is a property of the WHOLE response and text already on the wire cannot be recalled, so an incremental gate is impossible; buffering costs time-to-first-token (it now equals total latency) and that tradeoff is documented on the method. The alternative — leaving it ungated while the module docs claim a gate — was not acceptable, and refusing outright would have deleted a working public API. - The queue handler's `if (pipelineResult.suppressed)` arm is DELETED from the step-5b branch chain; it posts unconditionally with the pipeline's safe text. That removes the shadow-mode ordering bug by construction rather than reordering the branches. Escalation still fires on `confidenceScore < AI_CONFIDENCE.ESCALATE || pipelineResult.suppressed`. - apps/web/src/app/api/qa/route.ts streams `result.formatted.text` instead of `result.response`, so it consumes the published text and never needs to read `suppressed`. Side effect: the SSE stream now carries the web disclaimer and footer, which is the text we actually intend to publish. Tests (red-green verified against a temporary revert of each change): - pipeline-groundedness.test.ts (real generator via aimock + real formatter): a suppressed draft publishes the replacement and never the draft on all five platform targets (discord/github/slack/teams/web, Discord `parts` included); `result.response` still equals the draft; the escalated disclaimer is not stacked on the replacement; a grounded draft is published untouched; plus the streaming gate contract. - pipeline.test.ts: the formatter receives SUPPRESSED_RESPONSE_TEXT (not the draft) with `disclaimerText: AI_DISCLAIMER`; five generateStreamingResponse tests including one proving the WHOLE buffer is assessed, not per chunk. - queue ai-response.test.ts: posts the safe replacement rather than staying silent; shadow mode records a suppressed response (the bug above); escalates on suppression at score 0.95 asserting on THAT call's return value — the previous version of this test asserted on a stale earlier result. - qa-api.test.ts: streams the safe published text, never the suppressed draft. Assertions reassemble the SSE token events first — the route emits 8-char chunks, so a `not.toContain` on the raw payload passes vacuously. Verification: `npx vitest run --root packages/outpost --reporter=dot` 893 passed / 56 files; `npx vitest run` in apps/web 503 passed / 45 files; `npx tsc --noEmit -p packages/outpost/ai` clean (also queue and apps/web). Call-site enumeration (grep -rn across packages/ and apps/, dist excluded) SUPPRESSED_RESPONSE_TEXT (ADDED) packages/outpost/ai/src/pipeline.ts:38 declaration packages/outpost/ai/src/pipeline.ts:218 non-streaming swap packages/outpost/ai/src/pipeline.ts:348 streaming swap packages/outpost/ai/src/index.ts:19 public re-export packages/outpost/ai/src/pipeline.test.ts:19,257,276,511,526 packages/outpost/ai/src/pipeline-groundedness.test.ts:20,198,216,245,272 referenced in prose: ai/src/types.ts:215, queue/src/handlers/ai-response.ts:15,184, queue/src/__tests__/ai-response.test.ts:158 No non-test runtime consumer outside the ai package — consumers inherit the copy through `formatted`, they do not import the constant. PipelineResult (CHANGED — semantics of `response`/`formatted`/`suppressed`) packages/outpost/ai/src/types.ts:206 declaration packages/outpost/ai/src/pipeline.ts:4,102 only typed usage Structural consumers (no type import, so no compile-time coupling): packages/outpost/queue/src/handlers/ai-response.ts:121 (generateSupportResponse) - :162 reads .response → BOT Message content (the draft, intended) - :175 reads .formatted.text → ticket.suggestedResponse (now safe copy) - :192 reads .suppressed → log only, no longer a gate - :206 reads .formatted.text → shadow-mode SYSTEM message - :247 reads .formatted → adapter.postResponse - :272,276,296,308,309 read .suppressed → escalation + result payload apps/web/src/app/api/qa/route.ts:62 (generateSupportResponse) - :78 reads .formatted.text (was .response) → SSE token stream - reads .confidenceLevel, .searchResults for the metadata event No other file in packages/ or apps/ consumes a PipelineResult. AIPipeline.generateSupportResponse (CHANGED — `formatted` now gated) packages/outpost/ai/src/pipeline.ts:99 definition packages/outpost/queue/src/handlers/ai-response.ts:121 apps/web/src/app/api/qa/route.ts:62 tests: ai/src/pipeline.test.ts, ai/src/pipeline-groundedness.test.ts, queue/src/__tests__/ai-response.test.ts (mocked), apps/web/src/__tests__/qa-api.test.ts (mocked) AIPipeline.generateStreamingResponse (CHANGED — buffers, assesses, gates) packages/outpost/ai/src/pipeline.ts:310 definition No production caller anywhere in packages/ or apps/ — the only callers are the new tests (ai/src/pipeline.test.ts:496,508,523,534,552 and ai/src/pipeline-groundedness.test.ts:269,285). The web QA route uses the non-streaming path and chunks the finished text itself. ResponseGenerator.generateStream (UNCHANGED) packages/outpost/ai/src/generator.ts:172 definition packages/outpost/ai/src/pipeline.ts:333 sole runtime caller packages/outpost/ai/src/generator.test.ts:252,268 Raw model access below the gate, like the Anthropic client itself. AI_DISCLAIMER (UNCHANGED, new consumer) packages/outpost/ai/src/formatter.ts:22 declaration packages/outpost/ai/src/pipeline.ts:17,234 NEW import + use packages/outpost/ai/src/index.ts:14, formatter.test.ts:3,11,17,91, pipeline.test.ts:20 (new), pipeline-groundedness.test.ts:23 (new) Nothing was removed from any public surface. --- apps/web/src/__tests__/qa-api.test.ts | 70 +++++++++ apps/web/src/app/api/qa/route.ts | 11 +- packages/outpost/ai/src/index.ts | 2 +- .../ai/src/pipeline-groundedness.test.ts | 129 ++++++++++++++++- packages/outpost/ai/src/pipeline.test.ts | 135 +++++++++++++++++- packages/outpost/ai/src/pipeline.ts | 91 ++++++++++-- packages/outpost/ai/src/types.ts | 21 ++- .../queue/src/__tests__/ai-response.test.ts | 105 ++++++++++---- .../outpost/queue/src/handlers/ai-response.ts | 37 +++-- 9 files changed, 542 insertions(+), 59 deletions(-) diff --git a/apps/web/src/__tests__/qa-api.test.ts b/apps/web/src/__tests__/qa-api.test.ts index 10838a08..87aabb12 100644 --- a/apps/web/src/__tests__/qa-api.test.ts +++ b/apps/web/src/__tests__/qa-api.test.ts @@ -53,6 +53,24 @@ async function readStream(response: Response): Promise { return result; } +/** + * Reassemble the streamed answer from its token events. + * + * The route emits 8-character chunks, so asserting on the raw SSE payload cannot + * detect leaked text — any word longer than 8 chars is split across events and a + * `not.toContain` would pass vacuously. Join the tokens first. + */ +function tokenText(streamText: string): string { + return streamText + .split('\n\n') + .map((line) => line.replace(/^data: /, '').trim()) + .filter((data) => data && data !== '[DONE]') + .map((data) => JSON.parse(data) as { type: string; text?: string }) + .filter((event) => event.type === 'token') + .map((event) => event.text ?? '') + .join(''); +} + describe('POST /api/qa', () => { beforeEach(() => { mockGenerateSupportResponse.mockReset(); @@ -84,6 +102,7 @@ describe('POST /api/qa', () => { it('calls pipeline and streams response', async () => { mockGenerateSupportResponse.mockResolvedValue({ response: 'CopilotKit is great.', + formatted: { text: 'CopilotKit is great.', truncated: false }, confidenceLevel: 'HIGH', confidenceScore: 0.92, searchResults: [ @@ -118,6 +137,7 @@ describe('POST /api/qa', () => { it('passes conversation history to pipeline', async () => { mockGenerateSupportResponse.mockResolvedValue({ response: 'Follow up answer.', + formatted: { text: 'Follow up answer.', truncated: false }, confidenceLevel: 'MEDIUM', confidenceScore: 0.6, searchResults: [], @@ -147,6 +167,7 @@ describe('POST /api/qa', () => { it('cleans up pipeline after response', async () => { mockGenerateSupportResponse.mockResolvedValue({ response: 'Test.', + formatted: { text: 'Test.', truncated: false }, confidenceLevel: 'HIGH', confidenceScore: 0.9, searchResults: [], @@ -160,6 +181,55 @@ describe('POST /api/qa', () => { expect(mockDestroy).toHaveBeenCalled(); }); + // This route is a CONSUMER of the pipeline, and it inherits the groundedness + // gate rather than re-implementing it: it streams `formatted.text`, which the + // pipeline has already swapped for safe copy when the draft is suppressed. It + // must never stream `response` — that field intentionally still holds the + // ungrounded draft so a human handling the escalation can work from it. + it('streams the safe published text, never the suppressed draft', async () => { + mockGenerateSupportResponse.mockResolvedValue({ + response: '## Bug Confirmed\n\nOverride `.copilotKitInputControls`.', + formatted: { + text: "I couldn't find an answer to this, so I've escalated it.", + truncated: false, + }, + confidenceLevel: 'LOW', + confidenceScore: 0.39, + searchResults: [], + tokenUsage: { inputTokens: 10, outputTokens: 5 }, + latencyMs: 100, + suppressed: true, + }); + + const response = await POST(makeRequest({ question: 'is this a bug?' })); + const answer = tokenText(await readStream(response)); + + expect(answer).toBe("I couldn't find an answer to this, so I've escalated it."); + expect(answer).not.toContain('Bug Confirmed'); + expect(answer).not.toContain('copilotKitInputControls'); + }); + + it('streams the formatted text (footer and all), not the raw draft', async () => { + mockGenerateSupportResponse.mockResolvedValue({ + response: 'Use the `input` prop.', + formatted: { + text: 'Use the `input` prop.\n\n---\n*Powered by CopilotKit AI*', + truncated: false, + }, + confidenceLevel: 'HIGH', + confidenceScore: 0.9, + searchResults: [], + tokenUsage: { inputTokens: 10, outputTokens: 5 }, + latencyMs: 100, + suppressed: false, + }); + + const response = await POST(makeRequest({ question: 'how?' })); + const answer = tokenText(await readStream(response)); + + expect(answer).toBe('Use the `input` prop.\n\n---\n*Powered by CopilotKit AI*'); + }); + it('handles pipeline errors gracefully', async () => { mockGenerateSupportResponse.mockRejectedValue(new Error('Claude API timeout')); diff --git a/apps/web/src/app/api/qa/route.ts b/apps/web/src/app/api/qa/route.ts index cc4909b9..d253d54b 100644 --- a/apps/web/src/app/api/qa/route.ts +++ b/apps/web/src/app/api/qa/route.ts @@ -67,8 +67,15 @@ export async function POST(request: Request) { }, ); - // Stream the response text in chunks - const text = result.response; + // Stream the PUBLISHED text, not `result.response`. + // + // `response` is the model's raw draft and is internal — when the + // pipeline's groundedness gate suppresses it, `formatted` carries + // safe replacement copy while `response` still holds the draft + // for a human. Streaming `formatted.text` means this route + // inherits the gate instead of re-implementing it, so it never + // needs to read `suppressed`. + const text = result.formatted.text; const chunkSize = 8; for (let i = 0; i < text.length; i += chunkSize) { diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 6a94e2a3..75857e14 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -16,7 +16,7 @@ export { AI_DISCLAIMER_REVIEWED, ResponseFormatter, } from './formatter.js'; -export { AIPipeline } from './pipeline.js'; +export { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; export { analyzeSentiment } from './sentiment.js'; export { scoreEngagement } from './engagement.js'; export { getSentimentTrend } from './sentiment-trend.js'; diff --git a/packages/outpost/ai/src/pipeline-groundedness.test.ts b/packages/outpost/ai/src/pipeline-groundedness.test.ts index c20f83d6..5c2616ee 100644 --- a/packages/outpost/ai/src/pipeline-groundedness.test.ts +++ b/packages/outpost/ai/src/pipeline-groundedness.test.ts @@ -17,9 +17,13 @@ vi.mock('./config.js', () => ({ validateConfig: vi.fn(), })); -import { AIPipeline } from './pipeline.js'; +import { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; import { ResponseGenerator } from './generator.js'; -import { ResponseFormatter } from './formatter.js'; +import { + AI_DISCLAIMER, + AI_DISCLAIMER_ESCALATED, + ResponseFormatter, +} from './formatter.js'; import { assessGroundedness } from './groundedness.js'; import { ConfidenceLevel } from './types.js'; import type { SearchResult } from './types.js'; @@ -78,6 +82,10 @@ const SCORER_SCORE = 0.95; /** One invented identifier → penalty 0.15, and NOT suppressed (suppress needs 2). */ const UNGROUNDED_RESPONSE = 'Override `.copilotKitInputControls` to force compact mode.'; +/** Two invented identifiers → suppress. */ +const SUPPRESSED_DRAFT = + 'Override `.copilotKitInputControls` and `.copilotKitInputControlsExpanded`.'; + function createPipeline() { return new AIPipeline({ pathfinder: { @@ -142,9 +150,8 @@ describe('groundedness penalty is applied exactly once', () => { }); it('still clamps a suppressed response below the escalation gate', async () => { - // Two invented identifiers → suppress. mock.onMessage(/./, { - content: 'Override `.copilotKitInputControls` and `.copilotKitInputControlsExpanded`.', + content: SUPPRESSED_DRAFT, usage: { input_tokens: 100, output_tokens: 50 }, }); @@ -168,3 +175,117 @@ describe('groundedness penalty is applied exactly once', () => { expect(result.confidenceScore).toBeCloseTo(SCORER_SCORE, 5); }); }); + +/** + * Suppression is enforced at the boundary, so it must hold for EVERY platform + * target — a per-consumer check could only ever cover the consumers someone + * remembered. These run the real formatter, so they assert the bytes that would + * actually reach a thread, per platform. + */ +describe('suppression is enforced at the pipeline boundary', () => { + const PLATFORMS = ['discord', 'github', 'slack', 'teams', 'web'] as const; + + it.each(PLATFORMS)('publishes the replacement, never the draft, on %s', async (source) => { + mock.onMessage(/./, { + content: SUPPRESSED_DRAFT, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source }); + + expect(result.suppressed).toBe(true); + expect(result.formatted.text).toContain(SUPPRESSED_RESPONSE_TEXT); + expect(result.formatted.text).not.toContain('copilotKitInputControls'); + // Discord may split into parts — none of them may carry the draft either. + for (const part of result.formatted.parts ?? []) { + expect(part).not.toContain('copilotKitInputControls'); + } + }); + + it('keeps the original draft on result.response for the human escalation', async () => { + mock.onMessage(/./, { + content: SUPPRESSED_DRAFT, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.response).toBe(SUPPRESSED_DRAFT); + expect(result.response).not.toContain(SUPPRESSED_RESPONSE_TEXT); + }); + + it('does not stack the escalated disclaimer on copy that already promises a human', async () => { + mock.onMessage(/./, { + content: SUPPRESSED_DRAFT, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.formatted.text).toContain(AI_DISCLAIMER); + expect(result.formatted.text).not.toContain(AI_DISCLAIMER_ESCALATED); + expect(result.formatted.text).not.toContain('escalated this to our engineering team'); + }); + + it('publishes the model draft untouched when it is grounded', async () => { + const grounded = 'Use the `input` prop on CopilotChat to supply your own input component.'; + mock.onMessage(/./, { + content: grounded, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.suppressed).toBe(false); + expect(result.formatted.text).toContain(grounded); + expect(result.formatted.text).not.toContain(SUPPRESSED_RESPONSE_TEXT); + }); +}); + +/** + * The streaming entry point cannot gate incrementally, so it buffers, assesses, + * then yields. These pin that contract — the honest version of "the gate exists + * on this path too". + */ +describe('generateStreamingResponse gate', () => { + async function collect(stream: AsyncIterable): Promise { + const out: string[] = []; + for await (const chunk of stream) out.push(chunk); + return out; + } + + it('yields only the replacement text when the buffered draft is suppressed', async () => { + mock.onMessage(/./, { + content: SUPPRESSED_DRAFT, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'github' }), + ); + + expect(chunks).toEqual([SUPPRESSED_RESPONSE_TEXT]); + expect(chunks.join('')).not.toContain('copilotKitInputControls'); + }); + + it('yields the grounded draft in full', async () => { + const grounded = 'Use the `input` prop on CopilotChat to supply your own input component.'; + mock.onMessage(/./, { + content: grounded, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'github' }), + ); + + expect(chunks.length).toBeGreaterThan(0); + expect(chunks.join('')).toBe(grounded); + }); +}); diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 3ffce467..3ae3c60a 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -16,7 +16,8 @@ vi.mock('./config.js', () => ({ })); import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; -import { AIPipeline } from './pipeline.js'; +import { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; +import { AI_DISCLAIMER } from './formatter.js'; import { ConfidenceLevel, TicketPriority, TicketType } from './types.js'; import type { SearchResult, GeneratedResponse } from './types.js'; import type { ConfidenceAssessment } from './confidence.js'; @@ -244,6 +245,55 @@ describe('AIPipeline', () => { ]); }); + // The boundary: the draft must never reach the formatter (and so never + // reach `formatted`) when it is suppressed. Every consumer publishes + // `formatted`, so the swap here is what makes all of them safe. + it('hands the formatter the replacement copy, not the draft', async () => { + const draft = 'Override `.copilotKitGhostA` and `.copilotKitGhostB` to fix it.'; + mockGenerate.mockResolvedValue({ ...sampleGeneratedResponse, text: draft }); + + const result = await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(result.suppressed).toBe(true); + expect(mockFormat).toHaveBeenCalledWith( + SUPPRESSED_RESPONSE_TEXT, + 'github', + expect.any(Object), + ); + // ...while the draft stays on the result for the human escalation. + expect(result.response).toBe(draft); + }); + + // SUPPRESSED_RESPONSE_TEXT already promises a human follow-up, so the + // escalated variant would say it twice. + it('pairs the replacement with the plain disclaimer, not the escalated one', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Override `.copilotKitGhostA` and `.copilotKitGhostB` to fix it.', + }); + + await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(mockFormat).toHaveBeenCalledWith( + SUPPRESSED_RESPONSE_TEXT, + 'github', + expect.objectContaining({ + addDisclaimer: true, + disclaimerText: AI_DISCLAIMER, + }), + ); + }); + + it('leaves the draft as the published text when it is grounded', async () => { + await pipeline.generateSupportResponse('q', { source: 'github' }); + + expect(mockFormat).toHaveBeenCalledWith( + sampleGeneratedResponse.text, + 'github', + expect.any(Object), + ); + }); + // Withholding is driven by the objective signal only — identifiers no // retrieved source contains. Claim wording is fallible English, so it // buys a penalty and an escalation, never a withheld reply. @@ -470,6 +520,89 @@ describe('AIPipeline', () => { }); }); + // The streaming entry point cannot gate incrementally — you cannot know a draft + // invents an identifier until you have read it to the end, and text already on + // the wire cannot be recalled. So it buffers, assesses, then yields. These pin + // that contract; without it this method is the one ungated path to a thread. + describe('generateStreamingResponse', () => { + async function* streamOf(...chunks: string[]): AsyncIterable { + for (const chunk of chunks) yield chunk; + } + + async function collect(stream: AsyncIterable): Promise { + const out: string[] = []; + for await (const chunk of stream) out.push(chunk); + return out; + } + + it('yields the model chunks unchanged when the draft is grounded', async () => { + mockGenerateStream.mockReturnValue(streamOf('Use the ', '`useCopilotAction` ', 'hook.')); + + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'web' }), + ); + + expect(chunks).toEqual(['Use the ', '`useCopilotAction` ', 'hook.']); + }); + + it('yields ONLY the replacement copy when the buffered draft is suppressed', async () => { + mockGenerateStream.mockReturnValue( + streamOf('Override `.copilotKitInputControls` ', 'and `.copilotKitInputControlsExpanded`.'), + ); + + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'web' }), + ); + + expect(chunks).toEqual([SUPPRESSED_RESPONSE_TEXT]); + expect(chunks.join('')).not.toContain('copilotKitInputControls'); + }); + + it('assesses the WHOLE draft, not a single chunk (chunk-local text looks fine)', async () => { + // Split so no individual chunk carries both invented identifiers — a + // per-chunk gate would pass this through. + mockGenerateStream.mockReturnValue( + streamOf('Override `.copilotKitInputControls`', ' and also', ' `.copilotKitInputControlsExpanded`.'), + ); + + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'web' }), + ); + + expect(chunks).toEqual([SUPPRESSED_RESPONSE_TEXT]); + }); + + it('passes the retrieved sources and history through to the generator', async () => { + mockGenerateStream.mockReturnValue(streamOf('ok')); + const history = [{ role: 'user' as const, content: 'hi' }]; + + await collect( + pipeline.generateStreamingResponse('q', { + source: 'web', + conversationHistory: history, + }), + ); + + expect(mockGenerateStream).toHaveBeenCalledWith( + expect.objectContaining({ question: 'q', source: 'web' }), + sampleSearchResults, + history, + ); + }); + + it('still yields when Pathfinder is down (empty sources)', async () => { + mockSearchDocs.mockRejectedValueOnce(new Error('MCP down')); + mockGenerateStream.mockReturnValue(streamOf('best effort')); + + const chunks = await collect( + pipeline.generateStreamingResponse('q', { source: 'web' }), + ); + + expect(chunks).toEqual(['best effort']); + expect(mockGenerateStream).toHaveBeenCalledWith(expect.any(Object), [], undefined); + }); + }); + describe('classifyTicket', () => { it('should classify a ticket', async () => { mockClassify.mockResolvedValue({ diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index 15477f38..e8c69dd7 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -14,12 +14,30 @@ import { ResponseGenerator } from './generator.js'; import { ConfidenceScorer } from './confidence.js'; import { TicketClassifier } from './classifier.js'; import { + AI_DISCLAIMER, AI_DISCLAIMER_ESCALATED, AI_DISCLAIMER_REVIEWED, ResponseFormatter, } from './formatter.js'; import { validateConfig } from './config.js'; +/** + * The text published in place of a suppressed draft. + * + * The groundedness gate lives HERE, at the boundary where the response is + * produced, not at each consumer. When `groundedness.suppress` is true the + * pipeline swaps this copy into `formatted`, so every consumer — the queue + * handler, the web QA route, anything added later — publishes safe text without + * having to know the gate exists. The model's draft is still returned on + * `PipelineResult.response` for the human picking up the escalation. + * + * The copy promises a human follow-up itself, which is why callers pair it with + * the plain `AI_DISCLAIMER` rather than `AI_DISCLAIMER_ESCALATED` — stacking + * both would promise the same follow-up twice. + */ +export const SUPPRESSED_RESPONSE_TEXT = + "I couldn't find an answer to this in the CopilotKit or AG-UI documentation or source code, so I don't want to guess. I've escalated this to our team — someone will follow up in this thread."; + /** * Highest confidence score that still classifies BELOW HIGH. A degraded * confidence signal is clamped to this so it keeps its disclaimer and is never @@ -41,6 +59,14 @@ const SUPPRESSED_CONFIDENCE_CAP = AI_CONFIDENCE.ESCALATE - 0.01; * scoring (against the real generated response) → response formatting. Every * step has error handling — the pipeline never crashes, always returns a * graceful fallback. + * + * The groundedness gate is enforced HERE, not by consumers. Both entry points + * withhold an ungrounded draft themselves: `generateSupportResponse` swaps + * SUPPRESSED_RESPONSE_TEXT into `formatted`, and `generateStreamingResponse` + * buffers before yielding so it can do the same. Publishing what the pipeline + * hands back is therefore always safe — a consumer never has to read + * `suppressed` to avoid posting a fabrication. `suppressed` and `groundedness` + * remain on the result for analytics and escalation routing. */ export class AIPipeline { private pathfinder: PathfinderClient; @@ -184,6 +210,14 @@ export class AIPipeline { // Step 4: Format for target platform. // + // THE GATE. A suppressed draft never reaches `formatted`, so the withheld + // text cannot leak through any consumer — publishing `formatted` is + // always safe by construction. `response` below still carries the draft + // for the human handling the escalation. + const publishedText = groundedness.suppress + ? SUPPRESSED_RESPONSE_TEXT + : generatedResponse.text; + // The "we've escalated this" copy must be gated on the SAME condition the // worker uses to actually enqueue the ESCALATION job — score < ESCALATE // (see queue handlers/ai-response.ts) — NOT on the LOW *level* (score < @@ -191,15 +225,21 @@ export class AIPipeline { // is LOW but never escalated, so the reporter is promised a follow-up // that never comes. // - // Neither variant may hedge about the response's completeness — see the + // A suppressed response is the exception: SUPPRESSED_RESPONSE_TEXT already + // promises the same follow-up, so it takes the plain sentence instead of + // saying it twice. + // + // No variant may hedge about the response's completeness — see the // AI_DISCLAIMER doc comment in formatter.ts. const needsDisclaimer = finalConfidence !== ConfidenceLevel.HIGH; const willEscalate = finalConfidenceScore < AI_CONFIDENCE.ESCALATE; - const disclaimerText = willEscalate - ? AI_DISCLAIMER_ESCALATED - : AI_DISCLAIMER_REVIEWED; + const disclaimerText = groundedness.suppress + ? AI_DISCLAIMER + : willEscalate + ? AI_DISCLAIMER_ESCALATED + : AI_DISCLAIMER_REVIEWED; - const formatted = this.formatter.format(generatedResponse.text, options.source, { + const formatted = this.formatter.format(publishedText, options.source, { addDisclaimer: needsDisclaimer, disclaimerText, }); @@ -213,6 +253,8 @@ export class AIPipeline { } return { + // The ORIGINAL draft, even when suppressed — the human picking up the + // escalation works from it. Never publish this; publish `formatted`. response: generatedResponse.text, formatted, confidenceLevel: finalConfidence, @@ -245,7 +287,25 @@ export class AIPipeline { } /** - * Generate a streaming response. Yields text chunks as they arrive. + * Generate a response as a chunk stream, gated on groundedness. + * + * NOT incremental. The groundedness gate is a property of the WHOLE response + * — you cannot know a draft invents an identifier until you have read it to + * the end — so this method drains the model stream into a buffer, assesses it, + * and only then yields. Consumers get the same chunk boundaries the model + * produced, but they get them after generation completes: time-to-first-token + * equals total latency. + * + * That is the deliberate tradeoff. The alternative — yielding chunks as they + * arrive — cannot be gated at all: text already written to the wire cannot be + * withheld, and this entry point would be the one ungated way to reach a + * public thread. Callers that genuinely need incremental delivery must not use + * a gated pipeline; callers that want the metadata (confidence, sources, + * suppression) should use {@link generateSupportResponse} directly. + * + * When the draft is suppressed, the ONLY thing yielded is + * {@link SUPPRESSED_RESPONSE_TEXT} — the draft is discarded, not returned, on + * this path. Use `generateSupportResponse` if you need the draft. */ async *generateStreamingResponse( question: string, @@ -270,11 +330,26 @@ export class AIPipeline { source: options.source, }; - yield* this.generator.generateStream( + // Buffer the whole draft — the gate needs the complete text. + const chunks: string[] = []; + for await (const chunk of this.generator.generateStream( pipelineContext, searchResults, options.conversationHistory, - ); + )) { + chunks.push(chunk); + } + + const groundedness = assessGroundedness(chunks.join(''), searchResults); + if (groundedness.suppress) { + console.warn( + `[Pipeline] Streamed response withheld from public post — ${groundedness.reasons.join('; ')}`, + ); + yield SUPPRESSED_RESPONSE_TEXT; + return; + } + + yield* chunks; } /** diff --git a/packages/outpost/ai/src/types.ts b/packages/outpost/ai/src/types.ts index 139b556f..951f4687 100644 --- a/packages/outpost/ai/src/types.ts +++ b/packages/outpost/ai/src/types.ts @@ -204,9 +204,18 @@ export interface FormattedResponse { } export interface PipelineResult { - /** The generated response text */ + /** + * The model's draft, always — including when `suppressed` is true. Internal + * only: it is what the human handling an escalation edits from. Never publish + * it to a user-facing surface; publish `formatted` instead. + */ response: string; - /** Formatted response for the target platform */ + /** + * The text to publish, formatted for the target platform. Safe by + * construction: when `suppressed` is true this holds SUPPRESSED_RESPONSE_TEXT + * rather than the draft, so a consumer that publishes it unconditionally + * cannot leak an ungrounded answer. + */ formatted: FormattedResponse; /** Confidence assessment */ confidenceLevel: ConfidenceLevel; @@ -221,9 +230,11 @@ export interface PipelineResult { /** Deterministic check of the response against its sources. */ groundedness: GroundednessAssessment; /** - * True when the response makes a claim we can't stand behind and must not be - * posted to the public thread. Callers escalate to a human instead. Mirrors - * `groundedness.suppress` — kept at the top level because it gates a post. + * True when the draft makes a claim we can't stand behind, so `formatted` + * carries the safe replacement instead of `response`. Mirrors + * `groundedness.suppress`. This is a SIGNAL, not a gate a consumer must + * enforce — the pipeline already withheld the text. Read it to escalate to a + * human, to log, or for analytics; you do not need it to post safely. */ suppressed: boolean; } diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index 8952f9df..c0375a0c 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -148,18 +148,30 @@ const highConfidenceResult = { suppressed: false, }; +/** The draft the groundedness check refuses to publish (see #6167). */ +const SUPPRESSED_DRAFT = + '## Bug Confirmed: Cursor Jump\n\nOverride `.copilotKitInputControls` and ' + + '`.copilotKitInputControlsExpanded`.'; + /** - * A response the groundedness check refuses to publish (see #6167). - * - * Withholding is driven solely by identifiers no retrieved source contains, and - * the bar is two — so this fixture carries the two invented class names #6167 - * shipped. The claim wording rides along as penalty only; it does not withhold. + * Stands in for the pipeline's safe replacement copy. The handler is agnostic to + * the wording — its contract is "publish `formatted`, whatever it is" — so this + * fixture only has to be distinguishable from the draft. The real copy is pinned + * by name (SUPPRESSED_RESPONSE_TEXT) in the AI package's pipeline tests. + */ +const SAFE_REPLACEMENT_FIXTURE = 'I could not find an answer, so I have escalated this.'; + +/** + * What the pipeline returns for a suppressed draft: `response` keeps the draft for + * the human, `formatted` already carries the safe replacement. */ const suppressedResult = { ...highConfidenceResult, - response: - '## Bug Confirmed: Cursor Jump\n\nOverride `.copilotKitInputControls` and ' + - '`.copilotKitInputControlsExpanded`.', + response: SUPPRESSED_DRAFT, + formatted: { + text: SAFE_REPLACEMENT_FIXTURE, + truncated: false, + }, confidenceLevel: 'LOW', confidenceScore: 0.32, groundedness: { @@ -321,9 +333,13 @@ describe('handleAiResponse', () => { }); // A suppressed response is one the groundedness check found unsupportable. - // Confidence never gated the post-back, so these three behaviors are the - // whole point: nothing reaches the reporter, a human is pulled in, and the - // draft survives on the ticket for that human to edit. + // + // The handler does NOT gate on suppression — the pipeline already swapped safe + // copy into `formatted`, so the handler posts unconditionally. That is what + // these tests pin: the reporter gets the safe replacement and never the draft, + // a human is pulled in regardless of score, and the draft survives on the + // ticket for that human. A `suppressed` branch here is what previously made + // shadow mode drop exactly these records. describe('suppressed (ungrounded) responses', () => { beforeEach(() => { mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); @@ -331,7 +347,7 @@ describe('handleAiResponse', () => { mockHasAdapter.mockReturnValue(true); }); - it('never posts to the source platform', async () => { + it('posts the safe replacement, never the draft', async () => { const result = await handleAiResponse( { ticketId: 'tkt-1', source: 'discord' }, makeContext(), @@ -339,25 +355,33 @@ describe('handleAiResponse', () => { expect(result.success).toBe(true); expect(result.data?.suppressed).toBe(true); - expect(mockPostResponse).not.toHaveBeenCalled(); + // Posts unconditionally — with the pipeline's safe text. + expect(mockPostResponse).toHaveBeenCalledWith( + expect.objectContaining({ id: 'tkt-1' }), + suppressedResult.formatted, + ); + const posted = mockPostResponse.mock.calls[0][1] as { text: string }; + expect(posted.text).toBe(SAFE_REPLACEMENT_FIXTURE); + expect(posted.text).not.toContain('Bug Confirmed'); + expect(posted.text).not.toContain('copilotKitInputControls'); }); it('escalates to a human even though the score is above ESCALATE', async () => { - const result = await handleAiResponse( - { ticketId: 'tkt-1', source: 'discord' }, - makeContext(), - ); - - // 0.32 would escalate on score alone, so prove it escalates on - // suppression by raising the score above the gate. - mockPrismaJob.create.mockClear(); + // 0.32 would escalate on score alone, so raise the score above the gate + // and prove the escalation comes from suppression. Assert on THIS call's + // return value — asserting on an earlier result would prove nothing. mockGenerateSupportResponse.mockResolvedValue({ ...suppressedResult, confidenceScore: 0.95, confidenceLevel: 'HIGH', }); - await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.data?.confidenceScore).toBe(0.95); expect(result.data?.escalated).toBe(true); expect(mockPrismaJob.create).toHaveBeenCalledWith( expect.objectContaining({ @@ -371,19 +395,52 @@ describe('handleAiResponse', () => { it('still persists the draft so a human can edit and send it', async () => { await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + // The draft — not the replacement — is the BOT message a human works from. expect(mockPrismaMessage.create).toHaveBeenCalledWith( expect.objectContaining({ - data: expect.objectContaining({ type: 'BOT', isAiGenerated: true }), + data: expect.objectContaining({ + type: 'BOT', + isAiGenerated: true, + content: SUPPRESSED_DRAFT, + }), }), ); + // suggestedResponse is what bots pick up, so it holds the safe text. expect(mockPrismaTicket.update).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ - suggestedResponse: expect.any(String), + suggestedResponse: SAFE_REPLACEMENT_FIXTURE, }), }), ); }); + + // The regression this whole branch chain rewrite exists for: the old + // `if (suppressed)` arm ran BEFORE the SHADOW_MODE arm, so in shadow mode a + // suppressed response produced no shadow record — the responses most worth + // studying were the only ones that stopped being logged. + it('records a shadow message in shadow mode', async () => { + const originalShadow = process.env.SHADOW_MODE; + try { + process.env.SHADOW_MODE = 'true'; + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(mockPostResponse).not.toHaveBeenCalled(); + const shadowCall = mockPrismaMessage.create.mock.calls.find( + (call: Array>>) => + call[0].data.author === 'outpost-shadow', + ); + expect(shadowCall).toBeDefined(); + expect(shadowCall![0].data.content).toBe(SAFE_REPLACEMENT_FIXTURE); + } finally { + process.env.SHADOW_MODE = originalShadow; + } + }); }); it('does not escalate when confidence is above ESCALATE threshold', async () => { diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 61279918..e7d44548 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -7,10 +7,13 @@ * 3. Classifying the ticket inline (priority, type, tags) * 4. Formatting the response for the source platform * 5. Persisting the AI response as a Message record - * 6. Enqueuing an ESCALATION job if confidence is too low + * 6. Enqueuing an ESCALATION job if confidence is too low, or if the pipeline + * suppressed an ungrounded draft * * The pipeline itself handles Pathfinder retrieval, Claude generation, - * confidence scoring, and platform-specific formatting. + * confidence scoring, platform-specific formatting, and the groundedness gate — + * so what it hands back is always safe to publish (see SUPPRESSED_RESPONSE_TEXT + * in packages/outpost/ai/src/pipeline.ts). This handler does not re-check it. */ import { prisma } from '@copilotkit/outpost/db'; @@ -173,22 +176,28 @@ export async function handleAiResponse( }, }); - // 5b. Post the response back to the source platform. + // 5b. Post the response back to the source platform — unconditionally. // - // A suppressed response is one the groundedness check found unsupportable — - // it confirms a bug, asserts a root cause, or names identifiers absent from - // every retrieved source. Confidence alone never gated the post (it only - // picks the disclaimer and fires escalation), so a low score would not have - // stopped a fabrication from reaching a public thread. This does. The draft - // is still persisted above and lands on the ticket as suggestedResponse, so - // a human can edit and send it. + // No suppression check here on purpose. The pipeline withholds an + // ungrounded draft at the boundary: `pipelineResult.formatted` already + // carries safe replacement copy whenever `suppressed` is true (see + // SUPPRESSED_RESPONSE_TEXT in packages/outpost/ai/src/pipeline.ts), so + // posting it is always correct. Re-gating it here is what previously made + // shadow mode drop the very records worth studying — the suppressed arm ran + // before the SHADOW_MODE arm, so nothing was logged. The draft itself is + // persisted as the BOT Message in step 5 for the human to edit (while + // suggestedResponse holds the publishable text bots pick up), and step 6 + // below escalates on suppression regardless of score. const ticketSource = ticket.source as TicketSource; if (pipelineResult.suppressed) { console.warn( - `[AI Response] Withholding response for ticket ${ticketId} — ` + - `${pipelineResult.groundedness.reasons.join('; ')}. Escalating to a human.`, + `[AI Response] Ungrounded draft withheld for ticket ${ticketId} — ` + + `${pipelineResult.groundedness.reasons.join('; ')}. ` + + `Publishing the safe replacement and escalating to a human.`, ); - } else if (process.env.SHADOW_MODE === 'true') { + } + + if (process.env.SHADOW_MODE === 'true') { try { await prisma.message.create({ data: { @@ -284,7 +293,7 @@ export async function handleAiResponse( console.log( `[AI Response] Ticket ${ticketId}: confidence=${pipelineResult.confidenceLevel} ` + `(${(pipelineResult.confidenceScore * 100).toFixed(0)}%), latency=${pipelineResult.latencyMs}ms` + - `${pipelineResult.suppressed ? ', response withheld' : ''}`, + `${pipelineResult.suppressed ? ', ungrounded draft withheld' : ''}`, ); return { From 2f501ed8d413777231c34827d3e5c1ede79a72db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 30 Jul 2026 06:51:43 -0400 Subject: [PATCH 40/83] fix(ai): drop the autoSend field and ground the generator's confidence level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ResponseGenerator.generate` set two public fields from a score that deliberately excludes the groundedness penalty. `assessConfidence` is retrieval-only by design — the pipeline is the single place that deducts, so the penalty is not double-counted — but `autoSend` and `confidenceLevel` were computed from it BEFORE the deduction. A response whose `groundedness.suppress` was true therefore came back with `autoSend: true` and `confidenceLevel: HIGH`. Two sources scoring 0.9 and 0.85 give a retrieval score of 0.975, which clears `AI_CONFIDENCE.AUTO_RESPOND` (0.9) and `HIGH_THRESHOLD` (0.8) regardless of how ungrounded the text is. Six reviewers reported it independently. `autoSend` had no consumer anywhere in the repo — the only non-test occurrences were its declaration and its two assignments (evidence below). Rather than ship a public boolean that lies and that everybody must keep correct for nobody, it is removed. The gate it purported to implement is `PipelineResult.suppressed` plus the score-based escalation in queue/handlers/ai-response.ts, both of which already work. `confidenceLevel` stays — it is a public claim about the response — and is now classified from the penalised value: retrieval score minus `groundedness.penalty`, clamped to `SUPPRESSED_CONFIDENCE_CAP` when the gate would withhold the text, exactly as the pipeline clamps its own score. The penalised value is LOCAL to the classification. `confidenceScore` is not mutated and stays retrieval-only, so it still feeds the pipeline's `min()` undeducted and the penalty is still charged exactly once. No second deduction is reintroduced. `SUPPRESSED_CONFIDENCE_CAP` moves from a private const in pipeline.ts to an export in types.ts so the generator and the pipeline cannot disagree about what a withheld response is worth. Tests: `pipeline-groundedness.test.ts` gains four cases against the real generator over aimock — a suppressible response on 0.975-scoring sources must classify LOW (it classified HIGH before this change), an ungrounded but publishable response keeps its penalised level, a grounded response on the same sources keeps HIGH, and `autoSend` is no longer a key on the returned object. Red-green confirmed: both new invariants failed against the prior code with `expected 'HIGH' to be 'LOW'` and `expected true to be false`. Call sites ---------- REMOVED `GeneratedResponse.autoSend` (was types.ts:52) - writer packages/outpost/ai/src/generator.ts:142 (success path) — deleted - writer packages/outpost/ai/src/generator.ts:158 (fallback path) — deleted - fixture packages/outpost/ai/src/pipeline.test.ts:76 — deleted - assert packages/outpost/ai/src/generator.test.ts:115 — deleted - assert packages/outpost/ai/src/generator.test.ts:129 — deleted - production readers: NONE (this is why it was removed) Post-removal grep (node_modules and dist excluded): $ grep -rn 'autoSend' packages/ apps/ packages/outpost/ai/src/pipeline-groundedness.test.ts:232: it('no longer exposes an autoSend field ... packages/outpost/ai/src/pipeline-groundedness.test.ts:244: expect('autoSend' in generated).toBe(false); Zero remaining references outside the regression test that pins the removal. The all-filetypes grep (no --include filter) returns the same two lines, so no JSON, SQL, or Prisma surface referenced it either. REMOVED private const `SUPPRESSED_CONFIDENCE_CAP` from pipeline.ts:35 - sole reader packages/outpost/ai/src/pipeline.ts:167 — now reads the types.ts export; value is unchanged (`AI_CONFIDENCE.ESCALATE - 0.01`) ADDED `SUPPRESSED_CONFIDENCE_CAP` (packages/outpost/ai/src/types.ts) - packages/outpost/ai/src/pipeline.ts:167 (suppressed-score clamp) - packages/outpost/ai/src/generator.ts (classifyGroundedConfidence) ADDED private `ResponseGenerator.classifyGroundedConfidence` - packages/outpost/ai/src/generator.ts, inside `generate()` — the only call site; private, so no external surface CHANGED semantics of `GeneratedResponse.confidenceLevel` - writer packages/outpost/ai/src/generator.ts (success + fallback) - production readers: NONE. `AIPipeline.generateSupportResponse` computes `PipelineResult.confidenceLevel` itself from `finalConfidenceScore` (pipeline.ts:218) and never reads the generator's field, so the queue handler (queue/src/handlers/ai-response.ts:164,285,294) and the web dashboard (apps/web) are unaffected. - test readers: generator.test.ts:77,100,114,128 and the new cases in pipeline-groundedness.test.ts — all pass unchanged or updated here. CHANGED doc comments on `GeneratedResponse.confidenceScore` and `.confidenceLevel` (types.ts) so the field docs describe what the code does. The old "Whether this response should be auto-sent" comment described a gate nothing implemented and is gone with its field. NOT CHANGED `AI_CONFIDENCE.AUTO_RESPOND` (shared/src/constants.ts:39). Removing autoSend leaves it with no reader in src, but it is part of the documented action-band set alongside SUGGEST/ESCALATE and is mirrored in queue and github-app test fixtures. Deleting a public shared constant is out of scope for this fix. Verification ------------ npx vitest run --root packages/outpost --reporter=dot -> 56 files passed, 878 tests passed npx tsc --noEmit -p packages/outpost/ai -> exit 0 npx tsc --noEmit -p packages/outpost/queue -> exit 0 Prettier reports the same four pre-existing deviations at HEAD as after this change (generator.ts, pipeline.ts, generator.test.ts, pipeline.test.ts), so nothing here regressed formatting; unrelated reformatting was deliberately not bundled in. --- packages/outpost/ai/src/generator.test.ts | 2 - packages/outpost/ai/src/generator.ts | 37 +++++++-- .../ai/src/pipeline-groundedness.test.ts | 78 ++++++++++++++++++- packages/outpost/ai/src/pipeline.test.ts | 1 - packages/outpost/ai/src/pipeline.ts | 8 +- packages/outpost/ai/src/types.ts | 28 ++++++- 6 files changed, 134 insertions(+), 20 deletions(-) diff --git a/packages/outpost/ai/src/generator.test.ts b/packages/outpost/ai/src/generator.test.ts index 59e5bec7..feb43a96 100644 --- a/packages/outpost/ai/src/generator.test.ts +++ b/packages/outpost/ai/src/generator.test.ts @@ -112,7 +112,6 @@ describe('ResponseGenerator', () => { ); expect(result.confidenceLevel).toBe(ConfidenceLevel.LOW); - expect(result.autoSend).toBe(false); }); it('should return graceful fallback on API error', async () => { @@ -126,7 +125,6 @@ describe('ResponseGenerator', () => { expect(result.text).toContain('unable to generate'); expect(result.confidenceScore).toBe(0); expect(result.confidenceLevel).toBe(ConfidenceLevel.LOW); - expect(result.autoSend).toBe(false); }); it('should include conversation history for follow-ups', async () => { diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index cd3d6e38..c38263b3 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -1,8 +1,8 @@ import Anthropic from '@anthropic-ai/sdk'; -import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import type { PlatformTarget } from '@copilotkit/outpost/shared'; import type { GeneratedResponse, PipelineContext, SearchResult, TokenUsage } from './types.js'; -import { ConfidenceLevel, classifyConfidence } from './types.js'; +import { ConfidenceLevel, SUPPRESSED_CONFIDENCE_CAP, classifyConfidence } from './types.js'; +import type { GroundednessAssessment } from './groundedness.js'; import { assessGroundedness } from './groundedness.js'; import { config } from './config.js'; @@ -128,18 +128,20 @@ export class ResponseGenerator { }; const confidenceScore = this.assessConfidence(sources); - const confidenceLevel = classifyConfidence(confidenceScore); const latencyMs = Date.now() - startTime; // Assessed here (the response and its sources are both in hand) and // applied by the pipeline — exactly once. const groundedness = assessGroundedness(responseText, sources); + const confidenceLevel = this.classifyGroundedConfidence( + confidenceScore, + groundedness, + ); return { text: responseText, confidenceScore, confidenceLevel, sources, - autoSend: confidenceScore >= AI_CONFIDENCE.AUTO_RESPOND, reasoning: `Based on ${sources.length} source(s) with avg relevance ${this.avgScore(sources).toFixed(2)}`, tokenUsage, latencyMs, @@ -155,7 +157,6 @@ export class ResponseGenerator { confidenceScore: 0, confidenceLevel: ConfidenceLevel.LOW, sources, - autoSend: false, reasoning: `Generation failed: ${error instanceof Error ? error.message : String(error)}`, tokenUsage: { inputTokens: 0, outputTokens: 0 }, latencyMs, @@ -255,6 +256,32 @@ export class ResponseGenerator { return Math.min(avgRelevance + sourceCountBonus, 1.0); } + /** + * Classify the confidence LEVEL we publish for this response. + * + * `retrievalScore` is retrieval-only by design, so classifying it directly + * announced HIGH for any answer built on good sources — including one the + * groundedness gate would withhold entirely (two sources at 0.9/0.85 score + * 0.975 no matter how fabricated the text is). The level is a claim about the + * *response*, so it is classified from the penalised value, and clamped for a + * suppressed response the same way the pipeline clamps its own score. + * + * The penalised value is LOCAL. It is never written back to `confidenceScore`, + * which must stay penalty-free because it feeds the pipeline's `min()` before + * the pipeline performs the one and only deduction. Deducting into the score + * here is the double-counting bug this module just fixed. + */ + private classifyGroundedConfidence( + retrievalScore: number, + groundedness: GroundednessAssessment, + ): ConfidenceLevel { + let score = Math.max(0, retrievalScore - groundedness.penalty); + if (groundedness.suppress) { + score = Math.min(score, SUPPRESSED_CONFIDENCE_CAP); + } + return classifyConfidence(score); + } + private avgScore(sources: SearchResult[]): number { if (sources.length === 0) return 0; return sources.reduce((sum, s) => sum + s.score, 0) / sources.length; diff --git a/packages/outpost/ai/src/pipeline-groundedness.test.ts b/packages/outpost/ai/src/pipeline-groundedness.test.ts index 5c2616ee..731529d5 100644 --- a/packages/outpost/ai/src/pipeline-groundedness.test.ts +++ b/packages/outpost/ai/src/pipeline-groundedness.test.ts @@ -25,7 +25,7 @@ import { ResponseFormatter, } from './formatter.js'; import { assessGroundedness } from './groundedness.js'; -import { ConfidenceLevel } from './types.js'; +import { ConfidenceLevel, classifyConfidence } from './types.js'; import type { SearchResult } from './types.js'; /** @@ -289,3 +289,79 @@ describe('generateStreamingResponse gate', () => { expect(chunks.join('')).toBe(grounded); }); }); + +/** + * `GeneratedResponse.confidenceLevel` is a public claim about THIS response, so it + * must not read HIGH for text the groundedness gate would withhold. The trap is + * that `confidenceScore` is retrieval-only by design (the pipeline owns the single + * deduction), and these sources score 0.975 — comfortably HIGH — no matter how + * ungrounded the generated text is. The level therefore has to be classified from + * the penalised value, without the score itself being touched. + */ +describe('generator confidence level respects groundedness', () => { + /** Two invented identifiers → suppress, penalty 0.30. */ + const SUPPRESSIBLE_RESPONSE = + 'Override `.copilotKitInputControls` and `.copilotKitInputControlsExpanded`.'; + + it('does not report HIGH for a suppressible response built on high-quality sources', async () => { + mock.onMessage(/./, { + content: SUPPRESSIBLE_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const generated = await generator.generate({ question: 'q' }, SOURCES); + + // The premise: retrieval alone would classify this HIGH. + expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + expect(classifyConfidence(generated.confidenceScore)).toBe(ConfidenceLevel.HIGH); + + expect(generated.groundedness?.suppress).toBe(true); + // A response the gate withholds is never a confident one, and is clamped + // below the escalation gate exactly as the pipeline clamps it. + expect(generated.confidenceLevel).toBe(ConfidenceLevel.LOW); + }); + + it('reports the penalised level for an ungrounded but publishable response', async () => { + // One invented identifier → penalty 0.15, not suppressed. + mock.onMessage(/./, { + content: UNGROUNDED_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const generated = await generator.generate({ question: 'q' }, SOURCES); + + // 0.975 − 0.15 = 0.825, still HIGH — the penalty is charged, not amplified. + expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + expect(generated.confidenceLevel).toBe(ConfidenceLevel.HIGH); + }); + + it('keeps HIGH for a grounded response on the same sources', async () => { + mock.onMessage(/./, { + content: 'Use the `input` prop on CopilotChat to supply your own input component.', + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const generated = await generator.generate({ question: 'q' }, SOURCES); + + expect(generated.groundedness?.penalty).toBe(0); + expect(generated.confidenceLevel).toBe(ConfidenceLevel.HIGH); + }); + + it('no longer exposes an autoSend field for callers to trust', async () => { + mock.onMessage(/./, { + content: SUPPRESSIBLE_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const generated = await generator.generate({ question: 'q' }, SOURCES); + + // The field was a public boolean computed from the pre-deduction score, so + // it read `true` for exactly this response. It had no reader in the repo; + // rather than keep a gate nothing implements, it is gone. + expect('autoSend' in generated).toBe(false); + }); +}); diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 3ae3c60a..515a32ba 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -74,7 +74,6 @@ const sampleGeneratedResponse: GeneratedResponse = { confidenceScore: 0.85, confidenceLevel: ConfidenceLevel.HIGH, sources: sampleSearchResults, - autoSend: false, reasoning: 'Based on 2 sources', tokenUsage: { inputTokens: 500, outputTokens: 100 }, latencyMs: 2000, diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index e8c69dd7..c0df9183 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -6,7 +6,7 @@ import type { TokenUsage, SearchResult, } from './types.js'; -import { ConfidenceLevel, classifyConfidence } from './types.js'; +import { ConfidenceLevel, SUPPRESSED_CONFIDENCE_CAP, classifyConfidence } from './types.js'; import { assessGroundedness } from './groundedness.js'; import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import { PathfinderClient } from './pathfinder.js'; @@ -46,12 +46,6 @@ export const SUPPRESSED_RESPONSE_TEXT = */ const DEGRADED_CONFIDENCE_CAP = AI_CONFIDENCE.HIGH_THRESHOLD - 0.01; -/** - * Highest score a suppressed (unpublishable) response may carry. Sits just below - * the escalation gate so a withheld answer always reads as needing a human. - */ -const SUPPRESSED_CONFIDENCE_CAP = AI_CONFIDENCE.ESCALATE - 0.01; - /** * Main entry point for the Outpost AI pipeline. * diff --git a/packages/outpost/ai/src/types.ts b/packages/outpost/ai/src/types.ts index 951f4687..c15e6fae 100644 --- a/packages/outpost/ai/src/types.ts +++ b/packages/outpost/ai/src/types.ts @@ -26,6 +26,16 @@ export function classifyConfidence(score: number): ConfidenceLevel { return ConfidenceLevel.LOW; } +/** + * Highest score a suppressed (unpublishable) response may carry. Sits just below + * the escalation gate so a withheld answer always reads as needing a human. + * + * Lives here rather than in pipeline.ts because the generator classifies its own + * `confidenceLevel` against the same clamp — two places must agree on what a + * withheld response is worth, so they read one constant. + */ +export const SUPPRESSED_CONFIDENCE_CAP = AI_CONFIDENCE.ESCALATE - 0.01; + export interface SearchResult { /** Title of the matched document or section */ title: string; @@ -42,14 +52,24 @@ export interface SearchResult { export interface GeneratedResponse { /** The generated response text */ text: string; - /** Confidence score from 0 to 1 */ + /** + * Retrieval-quality confidence from 0 to 1 — how good the sources were, NOT + * what the response did with them. The groundedness penalty is deliberately + * absent: see `groundedness` below. + */ confidenceScore: number; - /** Classified confidence level */ + /** + * Confidence in THIS response, classified from `confidenceScore` after the + * groundedness penalty is deducted and clamped to SUPPRESSED_CONFIDENCE_CAP + * when `groundedness.suppress` is set. It therefore reads lower than + * `classifyConfidence(confidenceScore)` for an ungrounded answer, and can never + * report HIGH for one the gate would withhold. The deduction is local to the + * classification — `confidenceScore` is left retrieval-only so the pipeline's + * `min()` still charges the penalty exactly once. + */ confidenceLevel: ConfidenceLevel; /** Search results used as context for generation */ sources: SearchResult[]; - /** Whether this response should be auto-sent */ - autoSend: boolean; /** Reasoning for the confidence assessment */ reasoning: string; /** Token usage for cost monitoring */ From da318fefec80f2ee1e8c6c22cc69a9b3399ff669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:30:43 -0400 Subject: [PATCH 41/83] test(ai,queue): repair guards that passed without testing their claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-2 test-only pass over the groundedness-gate branch. Every guard below was mutation-proven: the behaviour it names was removed from the source, the guard was confirmed to fail, the source was restored, and the guard was confirmed to pass. No production source behaviour changed. - pipeline.test.ts: the shared generated-response fixture omits `groundedness`, so every test drove the `generatedResponse.groundedness ?? assessGroundedness()` RECOMPUTE branch and nothing exercised the generator-supplied pass-through — the core invariant of the single-deduction fix. Adds a sentinel assessment the recompute could not produce, plus an explicit test for the fallback branch. - pipeline.test.ts: disclaimer rows. The score-0.95 row asserted on `disclaimerText` for a case where `addDisclaimer` is false, i.e. on copy that is never rendered; and the "neutral MEDIUM copy" row claimed a third variant that does not exist (both the LOW-not-escalated band and MEDIUM resolve to the same exported AI_DISCLAIMER_REVIEWED). Rows now read the whole format options object, pin copy by imported constant rather than string literal, and the HIGH case asserts the real fact — no disclaimer is rendered at all. - pipeline-groundedness.test.ts: the double-application guard hard-coded `SCORER_SCORE - penalty * 2` (0.65), which is not a value the double-count bug can produce (it produces 0.675), so the guard passed with the bug restored — only a neighbouring equality check caught it. The value is now derived from named constants, the guard lives in its own test so it is the only assertion that can fail, and the surrounding arithmetic comment is corrected. - groundedness.test.ts: the "tolerates sources with missing title or content" test passed EMPTY STRINGS, so the `?? ''` absence guards never executed. Now uses genuinely absent fields via a partial cast, asserts the resulting behaviour rather than only "does not throw", and the name matches what it does. - queue/__tests__/ai-response.test.ts: `process.env.SHADOW_MODE = originalShadow` stores the STRING "undefined" when the var was unset, leaking a defined env var into every later test. All three sites now go through a `restoreShadowMode` helper that deletes when the original was absent (the pattern already used in onboarding-digest.test.ts), and the helper itself is pinned by two tests. Verified holding at HEAD, no change needed: the "escalates even though the score is above ESCALATE" test already asserts its own call's return value with a 0.95 score, and fails when `|| pipelineResult.suppressed` is deleted from the handler. Suite: 56 files / 914 tests green (was 906). tsc --noEmit -p packages/outpost/ai clean. --- packages/outpost/ai/src/groundedness.test.ts | 20 +- .../ai/src/pipeline-groundedness.test.ts | 57 ++++-- packages/outpost/ai/src/pipeline.test.ts | 171 +++++++++++++++--- .../queue/src/__tests__/ai-response.test.ts | 60 +++++- 4 files changed, 262 insertions(+), 46 deletions(-) diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index 3cb4bf14..4f26f7e9 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -5,6 +5,7 @@ import { MAX_GROUNDEDNESS_PENALTY, SUPPRESS_AT_UNSOURCED_IDENTIFIERS, } from './groundedness.js'; +import type { GroundednessAssessment } from './groundedness.js'; import type { SearchResult } from './types.js'; const source = (content: string, title = 'CopilotChat'): SearchResult => ({ @@ -318,9 +319,22 @@ describe('assessGroundedness', () => { expect(noSources.unsourcedIdentifiers).toEqual(['copilotKitInput']); }); - it('tolerates sources with missing title or content', () => { - const partial = [{ title: '', content: '', score: 0.5 } as SearchResult]; - expect(() => assessGroundedness('Anything at all.', partial)).not.toThrow(); + // The `?? ''` coalesces in the haystack build exist for source rows whose + // optional fields are genuinely ABSENT — a present-but-empty `title: ''` never + // reaches the coalesce, so a fixture built that way asserts nothing about them. + // A partial cast is the only way to reproduce the real shape: a Pathfinder row + // for a page with no title has no `title` key at all. + it('tolerates a source whose title, content and sourceUrl are absent entirely', () => { + const missingFields = [{ score: 0.5 } as unknown as SearchResult]; + + let result: GroundednessAssessment | undefined; + expect(() => { + result = assessGroundedness('Use `.copilotKitInput` here.', missingFields); + }).not.toThrow(); + + // There is nothing to ground against, so the identifier is correctly reported + // unsourced — the absent fields contributed no matchable text to the haystack. + expect(result?.unsourcedIdentifiers).toEqual(['copilotKitInput']); }); // The exact response from CopilotKit/CopilotKit#6167, condensed. It is withheld diff --git a/packages/outpost/ai/src/pipeline-groundedness.test.ts b/packages/outpost/ai/src/pipeline-groundedness.test.ts index 731529d5..0c952d7f 100644 --- a/packages/outpost/ai/src/pipeline-groundedness.test.ts +++ b/packages/outpost/ai/src/pipeline-groundedness.test.ts @@ -19,11 +19,7 @@ vi.mock('./config.js', () => ({ import { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; import { ResponseGenerator } from './generator.js'; -import { - AI_DISCLAIMER, - AI_DISCLAIMER_ESCALATED, - ResponseFormatter, -} from './formatter.js'; +import { AI_DISCLAIMER, AI_DISCLAIMER_ESCALATED, ResponseFormatter } from './formatter.js'; import { assessGroundedness } from './groundedness.js'; import { ConfidenceLevel, classifyConfidence } from './types.js'; import type { SearchResult } from './types.js'; @@ -79,6 +75,14 @@ const SOURCES: SearchResult[] = [ const SCORER_SCORE = 0.95; +/** + * What `ResponseGenerator.assessConfidence` scores for SOURCES: avg(0.9, 0.85) = + * 0.875 plus the 0.10 two-source count bonus. Retrieval quality ONLY — the + * generator must never fold the groundedness penalty into this, because it is the + * left operand of the pipeline's `min()`. + */ +const GENERATOR_RETRIEVAL_SCORE = 0.975; + /** One invented identifier → penalty 0.15, and NOT suppressed (suppress needs 2). */ const UNGROUNDED_RESPONSE = 'Override `.copilotKitInputControls` to force compact mode.'; @@ -129,9 +133,34 @@ describe('groundedness penalty is applied exactly once', () => { // min(generator 0.975, scorer 0.95) = 0.95, minus one penalty of 0.15. expect(result.confidenceScore).toBeCloseTo(SCORER_SCORE - penalty, 5); + }); + + // The regression this file exists for, isolated into its own test so the guard + // is the ONLY assertion that can fail — a neighbouring equality check catching + // the bug first would leave the guard itself unproven (and it was: the old + // hard-coded `SCORER_SCORE - penalty * 2` = 0.65 is not a value the bug can + // produce, so that guard passed even with the double deduction restored). + it('does not deduct the penalty twice', async () => { + mock.onMessage(/./, { + content: UNGROUNDED_RESPONSE, + usage: { input_tokens: 100, output_tokens: 50 }, + }); + + const pipeline = createPipeline(); + const result = await pipeline.generateSupportResponse('how do I force compact mode?', { + source: 'github', + }); + + const { penalty } = assessGroundedness(UNGROUNDED_RESPONSE, SOURCES); + + // Derived, not hard-coded: if the generator ALSO deducted from its own + // confidenceScore, min() would see 0.975 − 0.15 = 0.825 rather than 0.975, + // so min(0.825, 0.95) = 0.825, and the pipeline's own deduction would land + // the result at 0.825 − 0.15 = 0.675. + const doubleApplied = Math.min(GENERATOR_RETRIEVAL_SCORE - penalty, SCORER_SCORE) - penalty; + expect(doubleApplied).toBeCloseTo(0.675, 5); - // The regression this file exists for: two deductions gave 0.80 − 0.15 = 0.65. - expect(result.confidenceScore).not.toBeCloseTo(SCORER_SCORE - penalty * 2, 5); + expect(result.confidenceScore).not.toBeCloseTo(doubleApplied, 5); }); it('leaves the generator score free of the penalty so min() stays meaningful', async () => { @@ -144,7 +173,7 @@ describe('groundedness penalty is applied exactly once', () => { const generated = await generator.generate({ question: 'q' }, SOURCES); // Retrieval quality only: avg 0.875 + 0.10 count bonus. - expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + expect(generated.confidenceScore).toBeCloseTo(GENERATOR_RETRIEVAL_SCORE, 5); // The assessment rides along for the pipeline to apply. expect(generated.groundedness?.penalty).toBeCloseTo(0.15, 5); }); @@ -265,9 +294,7 @@ describe('generateStreamingResponse gate', () => { }); const pipeline = createPipeline(); - const chunks = await collect( - pipeline.generateStreamingResponse('q', { source: 'github' }), - ); + const chunks = await collect(pipeline.generateStreamingResponse('q', { source: 'github' })); expect(chunks).toEqual([SUPPRESSED_RESPONSE_TEXT]); expect(chunks.join('')).not.toContain('copilotKitInputControls'); @@ -281,9 +308,7 @@ describe('generateStreamingResponse gate', () => { }); const pipeline = createPipeline(); - const chunks = await collect( - pipeline.generateStreamingResponse('q', { source: 'github' }), - ); + const chunks = await collect(pipeline.generateStreamingResponse('q', { source: 'github' })); expect(chunks.length).toBeGreaterThan(0); expect(chunks.join('')).toBe(grounded); @@ -313,7 +338,7 @@ describe('generator confidence level respects groundedness', () => { const generated = await generator.generate({ question: 'q' }, SOURCES); // The premise: retrieval alone would classify this HIGH. - expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + expect(generated.confidenceScore).toBeCloseTo(GENERATOR_RETRIEVAL_SCORE, 5); expect(classifyConfidence(generated.confidenceScore)).toBe(ConfidenceLevel.HIGH); expect(generated.groundedness?.suppress).toBe(true); @@ -333,7 +358,7 @@ describe('generator confidence level respects groundedness', () => { const generated = await generator.generate({ question: 'q' }, SOURCES); // 0.975 − 0.15 = 0.825, still HIGH — the penalty is charged, not amplified. - expect(generated.confidenceScore).toBeCloseTo(0.975, 5); + expect(generated.confidenceScore).toBeCloseTo(GENERATOR_RETRIEVAL_SCORE, 5); expect(generated.confidenceLevel).toBe(ConfidenceLevel.HIGH); }); diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 515a32ba..fa43accf 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -17,7 +17,7 @@ vi.mock('./config.js', () => ({ import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; import { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; -import { AI_DISCLAIMER } from './formatter.js'; +import { AI_DISCLAIMER, AI_DISCLAIMER_ESCALATED, AI_DISCLAIMER_REVIEWED } from './formatter.js'; import { ConfidenceLevel, TicketPriority, TicketType } from './types.js'; import type { SearchResult, GeneratedResponse } from './types.js'; import type { ConfidenceAssessment } from './confidence.js'; @@ -178,49 +178,165 @@ describe('AIPipeline', () => { // Regression for #115: the "we've escalated this" copy must appear iff // the worker would actually enqueue an ESCALATION job (score < ESCALATE // = 0.4), not merely because the level is LOW (score < 0.5). - const disclaimerFor = async (score: number): Promise => { + // + // Returns the WHOLE format options object, not just the copy: `disclaimerText` + // is passed to the formatter unconditionally but only rendered when + // `addDisclaimer` is true, so asserting on the text without also reading the + // flag can assert on copy that never reaches a thread. + const disclaimerOptsFor = async ( + score: number, + ): Promise<{ addDisclaimer: boolean; disclaimerText: string }> => { mockScore.mockResolvedValue({ ...sampleConfidence, score }); await pipeline.generateSupportResponse('q', { source: 'discord' }); - const opts = mockFormat.mock.calls.at(-1)?.[2] as { disclaimerText: string }; - return opts.disclaimerText; + return mockFormat.mock.calls.at(-1)?.[2] as { + addDisclaimer: boolean; + disclaimerText: string; + }; }; it('promises escalation only when the score is below the ESCALATE gate (0.4)', async () => { - const text = await disclaimerFor(0.3); - expect(text).toContain("We've escalated this to our engineering team"); + const opts = await disclaimerOptsFor(0.3); + expect(opts.addDisclaimer).toBe(true); + expect(opts.disclaimerText).toBe(AI_DISCLAIMER_ESCALATED); }); - it('does NOT promise escalation for the LOW-but-not-escalated band [0.4, 0.5)', async () => { - const text = await disclaimerFor(0.45); - // Was the bug: 0.45 is LOW but never escalated, so no false promise. - expect(text).not.toContain('escalated'); - expect(text).toContain('will review and follow up'); - }); + // There is no third "MEDIUM" variant: the LOW-but-not-escalated band and the + // MEDIUM band deliberately share AI_DISCLAIMER_REVIEWED, because the thing + // that distinguishes them (retrieval score) is not something the reporter can + // act on — what matters is only whether a human follow-up was promised. Pinned + // by constant so a future third variant has to be an explicit decision. + it('uses the same reviewed copy for the LOW-but-not-escalated band [0.4, 0.5) and MEDIUM', async () => { + const low = await disclaimerOptsFor(0.45); + const medium = await disclaimerOptsFor(0.6); - it('uses the neutral MEDIUM copy for scores in [0.5, 0.8)', async () => { - const text = await disclaimerFor(0.6); - expect(text).not.toContain('escalated'); - expect(text).toContain('A member of our team will review'); + // Was the bug: 0.45 is LOW but never escalated, so no false promise. + expect(low.addDisclaimer).toBe(true); + expect(low.disclaimerText).toBe(AI_DISCLAIMER_REVIEWED); + expect(medium.addDisclaimer).toBe(true); + expect(medium.disclaimerText).toBe(AI_DISCLAIMER_REVIEWED); + expect(low.disclaimerText).not.toContain('escalated'); }); // No externally-visible disclaimer may hedge about the response's own // completeness — that copy invites the reader to distrust an answer we // chose to post. Confidence is expressed by escalating, not by hedging. - it.each([0.1, 0.3, 0.45, 0.6, 0.95])( + // + // Only non-HIGH scores are listed: at HIGH nothing is rendered, so a row for + // it would be asserting on copy the reader never sees. The HIGH case is + // covered by its own "renders no disclaimer at all" test below. + it.each([0.1, 0.3, 0.45, 0.6])( 'never hedges about completeness at score %s', async (score) => { - const text = await disclaimerFor(score); - expect(text).not.toMatch( + const opts = await disclaimerOptsFor(score); + expect(opts.addDisclaimer).toBe(true); + expect(opts.disclaimerText).not.toMatch( /may be incomplete|might be incomplete|may not be accurate/i, ); - expect(text).toContain('This is an AI-generated response.'); + expect(opts.disclaimerText).toContain(AI_DISCLAIMER); }, ); + // Belt and braces on the copy itself, independent of any score: every + // variant that can be rendered is checked, so adding a fourth constant + // cannot introduce hedging copy unnoticed. + it.each([ + ['AI_DISCLAIMER', AI_DISCLAIMER], + ['AI_DISCLAIMER_ESCALATED', AI_DISCLAIMER_ESCALATED], + ['AI_DISCLAIMER_REVIEWED', AI_DISCLAIMER_REVIEWED], + ])('%s never hedges about completeness', (_name, text) => { + expect(text).not.toMatch(/may be incomplete|might be incomplete|may not be accurate/i); + expect(text).toContain(AI_DISCLAIMER); + }); + + // At HIGH the formatter is told not to render one, so there is no copy to + // assert on — that absence is the assertion. + it('renders no disclaimer at all for a HIGH score', async () => { + const opts = await disclaimerOptsFor(0.95); + // min(generator 0.85, scorer 0.95) = 0.85 → HIGH. + expect(opts.addDisclaimer).toBe(false); + }); + // The scores above measure retrieval quality; these measure whether the // answer stayed inside what was retrieved. Without this, a fabrication // inherits the score of a good docs match (how #6167 got posted). describe('groundedness', () => { + // `generatedResponse.groundedness ?? assessGroundedness(...)` has two + // branches and the shared fixture omits `groundedness`, so every OTHER + // test in this file drives the recompute fallback. The pass-through is + // the whole point of the single-deduction fix — the generator assesses + // once, the pipeline applies once — so it is pinned here with a sentinel + // assessment the recompute could not possibly produce for this text. + describe('uses the generator-supplied assessment when there is one', () => { + /** + * Deliberately impossible from `assessGroundedness('Here is how to + * use CopilotKit actions...', sampleSearchResults)`, which yields a + * zero penalty and no suppression. If the pipeline recomputes, none + * of these values survive. + */ + const SENTINEL_ASSESSMENT = { + penalty: 0.25, + unverifiedClaims: ['sentinel claim'], + unsourcedIdentifiers: ['copilotKitSentinel'], + hedgeCount: 7, + suppress: false, + reasons: ['sentinel reason'], + }; + + it('passes the generator assessment through untouched', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + groundedness: SENTINEL_ASSESSMENT, + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + }); + + expect(result.groundedness).toEqual(SENTINEL_ASSESSMENT); + // And it is the value actually APPLIED: 0.85 − 0.25 = 0.60. A + // recompute would leave the score at 0.85. + expect(result.confidenceScore).toBeCloseTo(0.6, 5); + }); + + it('honours a generator-supplied suppress flag the recompute would not set', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + groundedness: { ...SENTINEL_ASSESSMENT, suppress: true }, + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + }); + + // The text is perfectly grounded, so only the injected flag can + // produce this — proof the pipeline read the generator's verdict. + expect(result.suppressed).toBe(true); + expect(mockFormat).toHaveBeenCalledWith( + SUPPRESSED_RESPONSE_TEXT, + 'github', + expect.any(Object), + ); + }); + + it('recomputes when the generator supplies no assessment', async () => { + // The fallback branch, asserted explicitly rather than relied on + // implicitly: this fixture has no `groundedness` key. + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + text: 'Override `.copilotKitInputControls` to fix it.', + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + }); + + expect(result.groundedness.unsourcedIdentifiers).toEqual([ + 'copilotKitInputControls', + ]); + expect(result.groundedness.penalty).toBeCloseTo(0.15, 5); + }); + }); + it('leaves a grounded response unpenalized and publishable', async () => { const result = await pipeline.generateSupportResponse('q', { source: 'github' }); @@ -535,7 +651,9 @@ describe('AIPipeline', () => { } it('yields the model chunks unchanged when the draft is grounded', async () => { - mockGenerateStream.mockReturnValue(streamOf('Use the ', '`useCopilotAction` ', 'hook.')); + mockGenerateStream.mockReturnValue( + streamOf('Use the ', '`useCopilotAction` ', 'hook.'), + ); const chunks = await collect( pipeline.generateStreamingResponse('q', { source: 'web' }), @@ -546,7 +664,10 @@ describe('AIPipeline', () => { it('yields ONLY the replacement copy when the buffered draft is suppressed', async () => { mockGenerateStream.mockReturnValue( - streamOf('Override `.copilotKitInputControls` ', 'and `.copilotKitInputControlsExpanded`.'), + streamOf( + 'Override `.copilotKitInputControls` ', + 'and `.copilotKitInputControlsExpanded`.', + ), ); const chunks = await collect( @@ -561,7 +682,11 @@ describe('AIPipeline', () => { // Split so no individual chunk carries both invented identifiers — a // per-chunk gate would pass this through. mockGenerateStream.mockReturnValue( - streamOf('Override `.copilotKitInputControls`', ' and also', ' `.copilotKitInputControlsExpanded`.'), + streamOf( + 'Override `.copilotKitInputControls`', + ' and also', + ' `.copilotKitInputControlsExpanded`.', + ), ); const chunks = await collect( diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index c0375a0c..c29d02b1 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -5,7 +5,7 @@ * classification, message persistence, and escalation triggering. * All external dependencies (Prisma, AIPipeline, etc.) are mocked. */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import type { JobHandlerContext } from '../types.js'; // ─── Mock Setup ───────────────────────────────────────────────────────────── @@ -96,6 +96,23 @@ function makeContext(): JobHandlerContext { }; } +/** + * Put SHADOW_MODE back exactly as it was, including "it was never set". + * + * `process.env.SHADOW_MODE = original` cannot express absence — assigning + * `undefined` to an env var stores the STRING `"undefined"`, which is truthy for + * the handler's `process.env.SHADOW_MODE === 'true'`-style reads and, worse, + * leaks a *defined* var into every test that runs afterwards. Mirrors the + * delete-when-absent restore in onboarding-digest.test.ts. + */ +function restoreShadowMode(original: string | undefined): void { + if (original !== undefined) { + process.env.SHADOW_MODE = original; + } else { + delete process.env.SHADOW_MODE; + } +} + const sampleTicket = { id: 'tkt-1', displayId: 'TKT-0001', @@ -438,7 +455,7 @@ describe('handleAiResponse', () => { expect(shadowCall).toBeDefined(); expect(shadowCall![0].data.content).toBe(SAFE_REPLACEMENT_FIXTURE); } finally { - process.env.SHADOW_MODE = originalShadow; + restoreShadowMode(originalShadow); } }); }); @@ -694,7 +711,7 @@ describe('handleAiResponse', () => { ); expect(shadowMessageCall).toBeDefined(); } finally { - process.env.SHADOW_MODE = originalShadow; + restoreShadowMode(originalShadow); } }); @@ -744,7 +761,7 @@ describe('handleAiResponse', () => { expect(result.success).toBe(true); } finally { - process.env.SHADOW_MODE = originalShadow; + restoreShadowMode(originalShadow); } }); @@ -841,3 +858,38 @@ describe('handleAiResponse', () => { ); }); }); + +/** + * The shadow-mode tests above set SHADOW_MODE and hand it back in a `finally`. + * Getting the hand-back wrong does not fail those tests — it silently defines + * SHADOW_MODE for every test that runs afterwards, because assigning `undefined` + * to `process.env.X` stores the string `"undefined"`. So the restore itself is + * pinned here rather than left to trust. + */ +describe('restoreShadowMode', () => { + const beforeEachTest = process.env.SHADOW_MODE; + afterEach(() => { + restoreShadowMode(beforeEachTest); + }); + + it('unsets SHADOW_MODE entirely when it was never set', () => { + delete process.env.SHADOW_MODE; + const original = process.env.SHADOW_MODE; + process.env.SHADOW_MODE = 'true'; + + restoreShadowMode(original); + + expect('SHADOW_MODE' in process.env).toBe(false); + expect(process.env.SHADOW_MODE).toBeUndefined(); + }); + + it('puts the original value back when it was set', () => { + process.env.SHADOW_MODE = 'false'; + const original = process.env.SHADOW_MODE; + process.env.SHADOW_MODE = 'true'; + + restoreShadowMode(original); + + expect(process.env.SHADOW_MODE).toBe('false'); + }); +}); From 89cb3d54857f2b4eb157c8d22a39e89750b5c941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 30 Jul 2026 09:33:12 -0400 Subject: [PATCH 42/83] fix(ai): escalate every charged claim, not just those the arithmetic catches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The groundedness penalty is capped at MAX_GROUNDEDNESS_PENALTY (0.6), so a response with a perfect base score plus the maximum positive feedback calibration lands on exactly 1.0 - 0.6 = AI_CONFIDENCE.ESCALATE. The escalation gate tests `< ESCALATE`, so that worst case posted with a "we will review it" disclaimer and paged nobody. The suppression clamp used to cover it incidentally; once claim wording stopped driving suppression, the case was exposed. A charged claim is now clamped below the gate outright. The split is deliberate: claim wording never gates PUBLICATION (it is fallible English, and withholding a reply on a misread is the failure this branch spent three attempts on), but it is decisive for ESCALATION (the bot asserted something it cannot back, so a person looks at it). The existing calibration test documented the hole with `toBeLessThanOrEqual` and now pins the fix with a strict `<`. Two cases added: a perfect score with maximum boost, and a charged claim on a generator-SUPPLIED assessment, so the clamp is verified on both the recompute and pass-through branches. The pass-through sentinel is now claim-free by design, with a comment explaining why: a charged claim clamps the score, which would mask the penalty arithmetic that test exists to observe. Its identifier, hedge count and reason remain impossible for a recompute, so it keeps its distinguishing power. Red-green: reverting the clamp fails 3 tests in pipeline.test.ts; restoring it returns 916/916 green across packages/outpost. Typecheck clean. Call sites: SUPPRESSED_CONFIDENCE_CAP (types.ts:37) — 3 readers, all unchanged in meaning: generator.ts:280 (level classification), pipeline.ts:187 (suppression clamp), pipeline.ts new claim clamp. groundedness.unverifiedClaims — read in pipeline.ts (this clamp, reasons logging) and queue/handlers/ai-response.ts (escalation reason string); both still hold, the field's semantics are unchanged. --- packages/outpost/ai/src/pipeline.test.ts | 58 +++++++++++++++++++++--- packages/outpost/ai/src/pipeline.ts | 16 +++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index fa43accf..699b6ff6 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -275,7 +275,12 @@ describe('AIPipeline', () => { */ const SENTINEL_ASSESSMENT = { penalty: 0.25, - unverifiedClaims: ['sentinel claim'], + // Deliberately claim-free: a charged claim is clamped below the + // escalation gate, which would mask the penalty arithmetic this + // test exists to observe. The identifier, hedge count and reason + // are still impossible for a recompute of this text, so the + // sentinel keeps its distinguishing power. + unverifiedClaims: [], unsourcedIdentifiers: ['copilotKitSentinel'], hedgeCount: 7, suppress: false, @@ -298,6 +303,25 @@ describe('AIPipeline', () => { expect(result.confidenceScore).toBeCloseTo(0.6, 5); }); + // The supplied assessment also drives the escalation clamp, not just + // the arithmetic — a claim the generator charged must reach a human + // even though the pipeline never re-derived it. + it('clamps below the escalation gate on a supplied charged claim', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + groundedness: { + ...SENTINEL_ASSESSMENT, + unverifiedClaims: ['sentinel claim'], + }, + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + }); + + expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); + }); + it('honours a generator-supplied suppress flag the recompute would not set', async () => { mockGenerate.mockResolvedValue({ ...sampleGeneratedResponse, @@ -457,15 +481,37 @@ describe('AIPipeline', () => { confidenceCalibration: 0.15, }); - // 0.85 + 0.15 calibration = 1.0, minus the capped 0.6 penalty = 0.4. - // The boost cannot outrun the deduction, and the answer lands on the - // gate rather than above it. + // 0.85 + 0.15 calibration = 1.0, minus the capped 0.6 penalty = 0.4 — + // which is exactly the gate, and the gate tests `<`. Arithmetic alone + // would leave this worst case unescalated, so a charged claim is + // clamped below the gate outright. expect(withBoost.groundedness.penalty).toBeCloseTo(0.6, 5); - expect(withBoost.confidenceScore).toBeCloseTo(0.4, 5); - expect(withBoost.confidenceScore).toBeLessThanOrEqual(AI_CONFIDENCE.ESCALATE); + expect(withBoost.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); expect(withBoost.confidenceLevel).toBe(ConfidenceLevel.LOW); }); + // The knife-edge above is not a rounding curiosity: it is the ONLY case + // where a bot that asserted an unverifiable claim would page nobody. + it('escalates a charged claim even at a perfect score with maximum boost', async () => { + mockScore.mockResolvedValue({ ...sampleConfidence, score: 1 }); + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + confidenceScore: 1, + text: 'Bug Confirmed. Root cause is a re-render. The fix is trivial.', + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + confidenceCalibration: 0.2, + }); + + expect(result.groundedness.unverifiedClaims.length).toBeGreaterThan(0); + // Not withheld — claim wording never gates publication... + expect(result.suppressed).toBe(false); + // ...but it does guarantee a human sees it. + expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); + }); + // A boost cannot lift a withheld answer over the escalation gate either: // the suppression clamp sits below it by construction. it('keeps a suppressed response below the gate even with a positive boost', async () => { diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index c0df9183..e1ff1575 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -187,6 +187,22 @@ export class AIPipeline { finalConfidenceScore = Math.min(finalConfidenceScore, SUPPRESSED_CONFIDENCE_CAP); } + // An unverifiable claim always gets a human, whatever the arithmetic says. + // + // The penalty alone cannot guarantee that. The deduction is capped at + // MAX_GROUNDEDNESS_PENALTY (0.6), so a perfect base score plus the maximum + // positive feedback calibration lands on exactly 1.0 - 0.6 = ESCALATE, and + // the escalation gate tests `< ESCALATE` — the worst-case wording case + // would post with a "we'll review it" disclaimer and page nobody. + // + // Claims are penalty-only for the WITHHOLDING decision (see groundedness.ts + // — claim wording is fallible English and must never gate publication), but + // they are decisive for the ESCALATION decision: the bot asserted something + // it cannot back, so a person looks at it. + if (groundedness.unverifiedClaims.length > 0) { + finalConfidenceScore = Math.min(finalConfidenceScore, SUPPRESSED_CONFIDENCE_CAP); + } + // Safety cap: a DEGRADED confidence signal (LLM scorer unavailable → heuristic // fallback over Pathfinder's synthetic rank-scores) must never present as HIGH. // HIGH suppresses the disclaimer and is treated as authoritative, so an ungrounded From 3cf34b3bf69233f0ba186aa0da2e1752260f9f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:01:54 -0400 Subject: [PATCH 43/83] fix(ai): escalate only claims that assert our own verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on #143 (jerelvelarde, approved with six items). The escalation clamp fired on `unverifiedClaims.length > 0`, which included two patterns that a correct, docs-grounded answer uses in ordinary prose: "This is a known issue, fixed in 1.9.2." "The fix is to pass the `input` prop." Both are things the bot SHOULD say when the docs say them, and each one created an ESCALATION job plus the escalated disclaimer regardless of how good the answer was. That is a person's attention spent on English rather than on fabrication, and unlike the withholding trade-off it would have been paid every day. Claims are now split by who is being quoted. ESCALATION_FORCING_CATEGORIES holds the ones asserting that WE did something we cannot have done — confirmed a bug, established a root cause, reproduced it. `known-issue` and `fix` are new/moved categories that are priced but page nobody: the response still scores lower, so a wording-heavy answer still drifts toward review, it just does not wake anyone. `GroundednessAssessment` gains `forcesEscalation`, and the pipeline clamps on `suppress || forcesEscalation` rather than on any charged claim. Publication is unchanged — claim wording still never withholds anything. One thing did not work on the first attempt: `known` was in the real-bug pattern's adjective list, so "this is a known issue" matched both categories and escalated anyway, defeating the split for its single most common phrasing. Removed from that list, with a named regression test. Also from the review: - constants.ts: AUTO_RESPOND and SUGGEST are documented as feeding the pipeline. They have no reader in src/ — the action-based scheme they described was never implemented (everything posts; what varies is the disclaimer and whether a human is paged), and the last reader was autoSend, removed in 2f501ed. Doc now says so, and says to delete them rather than rewire them if a real auto-post gate is ever built. - types.ts: "the penalty is charged exactly once" was narrower than it read. True of the deterministic penalty; the LLM scorer also weighs groundedness as rubric factor 5 and enters through the same min() before this deduction. Named as intentional defense in depth instead of overstated as a single mechanism. - pipeline.ts: two consecutive if blocks clamped to the same cap. Collapsed. Kept deliberately: classifyGroundedConfidence guards GeneratedResponse .confidenceLevel, which has no production reader since AIPipeline recomputes the level. But ResponseGenerator is exported from index.ts, so that field is public API, and the alternative is shipping a public value that reads HIGH for a response the gate would withhold — the exact bug class autoSend was. Tests: escalates/does-not-escalate corpus over both claim kinds, the "known issue" regression guard, mixed-sentence case, and pipeline coverage for a supplied assessment on both sides of the split. Reverting the category split fails 5 tests. 929 tests green across packages/outpost, 10/10 packages, typecheck and build clean. Call sites: GroundednessAssessment.forcesEscalation (ADDED) — written in groundedness.ts assessGroundedness, read in pipeline.ts (the clamp); no other reader. ClaimCategory (CHANGED, +known-issue) — module-private, no external reference. AI_CONFIDENCE.AUTO_RESPOND / .SUGGEST (comment only, values unchanged) — grep confirms zero src/ readers; queue and github-app test fixtures still reference the numbers and are unaffected. --- packages/outpost/ai/src/groundedness.test.ts | 63 ++++++++++++++++ packages/outpost/ai/src/groundedness.ts | 78 ++++++++++++++++++-- packages/outpost/ai/src/pipeline.test.ts | 32 +++++++- packages/outpost/ai/src/pipeline.ts | 30 ++++---- packages/outpost/ai/src/types.ts | 9 ++- packages/outpost/shared/src/constants.ts | 19 +++-- 6 files changed, 201 insertions(+), 30 deletions(-) diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index 4f26f7e9..739136ad 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -187,6 +187,69 @@ describe('assessGroundedness', () => { expect(result.suppress).toBe(false); }); + // Every charged claim costs confidence, but only the ones asserting that WE + // verified something page a human. The split matters operationally: a claim + // phrase that a correct docs-grounded answer uses in normal prose would + // otherwise create an ESCALATION job several times a day, and the escalation + // queue is a person's attention. + describe('escalation is narrower than charging', () => { + it.each([ + 'Bug Confirmed: the cursor resets on every keystroke.', + 'I confirmed the bug on the latest version.', + 'This is a real bug worth fixing in the core.', + 'Root cause is a re-render on every keystroke.', + 'I reproduced this locally.', + 'We ran the tests and they fail.', + ])('escalates on own-verification claim %j', (response) => { + const result = assessGroundedness(response, CHAT_DOCS); + expect(result.forcesEscalation).toBe(true); + expect(result.penalty).toBeGreaterThan(0); + // Still published — claim wording never withholds. + expect(result.suppress).toBe(false); + }); + + it.each([ + // Both of these are things a correct, docs-grounded answer says. + 'This is a known issue, fixed in 1.9.2.', + 'The fix is to pass the `input` prop.', + 'This is a known issue. The fix is to pass the `input` prop.', + ])('charges but does NOT escalate on docs-reportable claim %j', (response) => { + const result = assessGroundedness(response, CHAT_DOCS); + expect(result.unverifiedClaims.length).toBeGreaterThan(0); + expect(result.penalty).toBeGreaterThan(0); + expect(result.forcesEscalation).toBe(false); + expect(result.suppress).toBe(false); + }); + + // Regression guard: `known` used to appear in the real-bug pattern's + // adjective list, so this sentence matched two categories and escalated + // anyway — defeating the whole split for the most common phrasing of it. + it('does not escalate "this is a known issue" via the real-bug pattern', () => { + const result = assessGroundedness('This is a known issue.', CHAT_DOCS); + expect(result.unverifiedClaims).toEqual(['claims a known bug']); + expect(result.forcesEscalation).toBe(false); + }); + + it('escalates when a docs-reportable claim sits beside an own-verification one', () => { + const result = assessGroundedness( + 'This is a known issue. I reproduced it locally.', + CHAT_DOCS, + ); + expect(result.forcesEscalation).toBe(true); + expect(result.reasons.join(' ')).toContain('asserts own verification'); + }); + + it('leaves a grounded answer with no claim at all alone', () => { + const result = assessGroundedness( + 'Use the `input` prop on CopilotChat to supply your own input component.', + CHAT_DOCS, + ); + expect(result.unverifiedClaims).toEqual([]); + expect(result.forcesEscalation).toBe(false); + expect(result.penalty).toBe(0); + }); + }); + // The reported basis has to equal what was actually billed, or the log line // understates the deduction it is supposed to explain. describe('reported reasons match what was charged', () => { diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts index c7c3cd87..9e0389da 100644 --- a/packages/outpost/ai/src/groundedness.ts +++ b/packages/outpost/ai/src/groundedness.ts @@ -28,10 +28,18 @@ import type { SearchResult } from './types.js'; * either in the sources we handed the model or it is not. No English is parsed * to reach it. * - * 2. **Claim phrases — penalty only.** "Bug confirmed", "root cause is", "I - * reproduced this" are still detected, still charged + * 2. **Claim phrases — never withhold; sometimes escalate.** "Bug confirmed", + * "root cause is", "I reproduced this" are still detected, still charged * `PENALTY_PER_UNVERIFIED_CLAIM`, and still reported on `unverifiedClaims` and - * `reasons`. They no longer contribute to `suppress` at all. + * `reasons`. They never contribute to `suppress`. The subset that asserts *our + * own* verification additionally sets `forcesEscalation`, which the pipeline + * clamps on so a person reviews the answer — but the answer still posts. + * + * That subset is narrower than "any claim phrase": "this is a known issue, + * fixed in 1.9.2" and "the fix is to pass the `input` prop" are ordinary + * sentences in a correct docs-grounded answer, so they are priced and left + * alone. Escalating on those would page a human several times a day for + * English rather than for fabrication. See ESCALATION_FORCING_CATEGORIES. * * Why: deciding whether a sentence *asserts* a claim or *denies* it is natural * language negation, and three successive regex attempts at it failed in three @@ -49,7 +57,33 @@ import type { SearchResult } from './types.js'; */ /** The kind of unsupportable claim a pattern detects. Several patterns can share one. */ -type ClaimCategory = 'confirmation' | 'bug-validity' | 'root-cause' | 'fix' | 'reproduction'; +type ClaimCategory = + | 'confirmation' + | 'bug-validity' + | 'known-issue' + | 'root-cause' + | 'fix' + | 'reproduction'; + +/** + * The categories that force an escalation, as opposed to only charging a penalty. + * + * The split is about WHO is being quoted. These four assert that WE did something + * we cannot have done — confirmed a bug, established its cause, reproduced it — so + * a person has to look at the answer. + * + * `known-issue` and `fix` are deliberately absent. "This is a known issue, fixed in + * 1.9.2" and "the fix is to pass the `input` prop" are things a correct, + * docs-grounded answer says all day, and forcing an escalation on each one would + * page a human for ordinary English. They still carry the penalty, so a response + * built out of them still scores lower; it just doesn't wake anyone up. + */ +const ESCALATION_FORCING_CATEGORIES: ReadonlySet = new Set([ + 'confirmation', + 'bug-validity', + 'root-cause', + 'reproduction', +]); /** * Claims of verification the bot cannot make: it has no repo, repro, or test run. @@ -77,11 +111,15 @@ const UNVERIFIED_CLAIM_PATTERNS: Array<{ { pattern: /\bknown\s+(?:bug|issue|regression)\b/i, label: 'claims a known bug', - category: 'bug-validity', + category: 'known-issue', }, { pattern: - /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|known|legitimate)\s+(?:bug|issue|regression|defect)\b/i, + /\bthis\s+is\s+(?:a|an)\s+(?:real|genuine|confirmed|legitimate)\s+(?:bug|issue|regression|defect)\b/i, + // `known` is deliberately NOT in the adjective list: "this is a known + // issue" belongs to the known-issue category, which is priced but does + // not page anyone. Leaving it here made that sentence match both + // categories and escalate anyway. label: 'asserts the report is a real bug', category: 'bug-validity', }, @@ -199,6 +237,17 @@ export interface GroundednessAssessment { * instead; a lowered score alone does not stop a post. */ suppress: boolean; + /** + * True when at least one charged claim asserts that WE verified something — + * confirmed a bug, established a root cause, reproduced it. The caller clamps + * below the escalation gate on this so a person reviews the answer. + * + * Distinct from `unverifiedClaims.length > 0`, which also counts claims a + * correct docs-grounded answer legitimately makes ("known issue", "the fix + * is"). Those are priced but do not page anyone — see + * ESCALATION_FORCING_CATEGORIES. + */ + forcesEscalation: boolean; /** Human-readable reasons, for logs and the dashboard. */ reasons: string[]; } @@ -295,6 +344,7 @@ export function assessGroundedness( unsourcedIdentifiers: [], hedgeCount: 0, suppress: false, + forcesEscalation: false, reasons: [], }; @@ -313,12 +363,14 @@ export function assessGroundedness( // withheld reply. const unverifiedClaims: string[] = []; const chargedCategories = new Set(); + const escalationForcing: string[] = []; for (const { pattern, label, category } of UNVERIFIED_CLAIM_PATTERNS) { if (chargedCategories.has(category)) continue; if (!pattern.test(normalized)) continue; chargedCategories.add(category); unverifiedClaims.push(label); + if (ESCALATION_FORCING_CATEGORIES.has(category)) escalationForcing.push(label); } // Sources are searched as one haystack: an identifier documented on any @@ -355,11 +407,15 @@ export function assessGroundedness( // Objective signal only. Claim wording is priced above and stops there. const suppress = unsourcedIdentifiers.length >= SUPPRESS_AT_UNSOURCED_IDENTIFIERS; + const forcesEscalation = escalationForcing.length > 0; const reasons: string[] = []; if (unverifiedClaims.length) { reasons.push(`unverifiable claims: ${unverifiedClaims.join(', ')}`); } + if (forcesEscalation) { + reasons.push(`asserts own verification: ${escalationForcing.join(', ')}`); + } if (unsourcedIdentifiers.length) { reasons.push(`identifiers absent from sources: ${unsourcedIdentifiers.join(', ')}`); } @@ -367,5 +423,13 @@ export function assessGroundedness( reasons.push(`${hedgeCount} hedge markers`); } - return { penalty, unverifiedClaims, unsourcedIdentifiers, hedgeCount, suppress, reasons }; + return { + penalty, + unverifiedClaims, + unsourcedIdentifiers, + hedgeCount, + suppress, + forcesEscalation, + reasons, + }; } diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 699b6ff6..051bfa0f 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -304,14 +304,15 @@ describe('AIPipeline', () => { }); // The supplied assessment also drives the escalation clamp, not just - // the arithmetic — a claim the generator charged must reach a human - // even though the pipeline never re-derived it. - it('clamps below the escalation gate on a supplied charged claim', async () => { + // the arithmetic — an own-verification claim the generator charged + // must reach a human even though the pipeline never re-derived it. + it('clamps below the escalation gate on a supplied own-verification claim', async () => { mockGenerate.mockResolvedValue({ ...sampleGeneratedResponse, groundedness: { ...SENTINEL_ASSESSMENT, - unverifiedClaims: ['sentinel claim'], + unverifiedClaims: ['claims to have reproduced or tested'], + forcesEscalation: true, }, }); @@ -322,6 +323,29 @@ describe('AIPipeline', () => { expect(result.confidenceScore).toBeLessThan(AI_CONFIDENCE.ESCALATE); }); + // The other half of that contract, and the reason the pipeline reads + // `forcesEscalation` rather than `unverifiedClaims.length`: a charged + // claim that is only reporting what the docs say gets priced and + // published, without paging anyone. + it('does not clamp on a charged claim that is not own-verification', async () => { + mockGenerate.mockResolvedValue({ + ...sampleGeneratedResponse, + groundedness: { + ...SENTINEL_ASSESSMENT, + unverifiedClaims: ['claims a known bug'], + forcesEscalation: false, + }, + }); + + const result = await pipeline.generateSupportResponse('q', { + source: 'github', + }); + + // 0.85 − 0.25 penalty = 0.60, well clear of the gate. + expect(result.confidenceScore).toBeCloseTo(0.6, 5); + expect(result.confidenceScore).toBeGreaterThan(AI_CONFIDENCE.ESCALATE); + }); + it('honours a generator-supplied suppress flag the recompute would not set', async () => { mockGenerate.mockResolvedValue({ ...sampleGeneratedResponse, diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index e1ff1575..3c490f31 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -183,23 +183,27 @@ export class AIPipeline { // LOW, and the worker's score-based escalation fires on its own. The // capped penalty alone can't guarantee this — a top score plus positive // calibration lands exactly ON the gate, which does not escalate. - if (groundedness.suppress) { - finalConfidenceScore = Math.min(finalConfidenceScore, SUPPRESSED_CONFIDENCE_CAP); - } - - // An unverifiable claim always gets a human, whatever the arithmetic says. + // Two independent reasons to guarantee a human sees this, both clamping to + // the same cap: + // + // 1. `suppress` — the response named identifiers no source contains, so the + // draft is withheld and the reporter gets the no-answer copy instead. + // 2. `forcesEscalation` — the response asserted that WE verified something + // (confirmed a bug, established a root cause, reproduced it). That text + // still publishes; claim wording is fallible English and must never gate + // publication. But somebody checks it. // - // The penalty alone cannot guarantee that. The deduction is capped at + // The penalty alone cannot guarantee either. The deduction is capped at // MAX_GROUNDEDNESS_PENALTY (0.6), so a perfect base score plus the maximum // positive feedback calibration lands on exactly 1.0 - 0.6 = ESCALATE, and - // the escalation gate tests `< ESCALATE` — the worst-case wording case - // would post with a "we'll review it" disclaimer and page nobody. + // the escalation gate tests `< ESCALATE` — the worst case would post with a + // "we'll review it" disclaimer and page nobody. // - // Claims are penalty-only for the WITHHOLDING decision (see groundedness.ts - // — claim wording is fallible English and must never gate publication), but - // they are decisive for the ESCALATION decision: the bot asserted something - // it cannot back, so a person looks at it. - if (groundedness.unverifiedClaims.length > 0) { + // `forcesEscalation` is deliberately narrower than `unverifiedClaims.length + // > 0`: "this is a known issue, fixed in 1.9.2" and "the fix is to pass the + // `input` prop" are ordinary sentences in a correct docs-grounded answer. + // They are priced, not escalated. See ESCALATION_FORCING_CATEGORIES. + if (groundedness.suppress || groundedness.forcesEscalation) { finalConfidenceScore = Math.min(finalConfidenceScore, SUPPRESSED_CONFIDENCE_CAP); } diff --git a/packages/outpost/ai/src/types.ts b/packages/outpost/ai/src/types.ts index c15e6fae..7cb79053 100644 --- a/packages/outpost/ai/src/types.ts +++ b/packages/outpost/ai/src/types.ts @@ -65,7 +65,14 @@ export interface GeneratedResponse { * `classifyConfidence(confidenceScore)` for an ungrounded answer, and can never * report HIGH for one the gate would withhold. The deduction is local to the * classification — `confidenceScore` is left retrieval-only so the pipeline's - * `min()` still charges the penalty exactly once. + * `min()` still charges the deterministic penalty exactly once. + * + * "Exactly once" is about THIS penalty, not about groundedness overall. The + * LLM confidence scorer also weighs groundedness (rubric factor 5 in + * confidence.ts), and its score enters through the same `min()` before this + * deduction — so an ungrounded answer can be marked down by two independent + * mechanisms. That is intended as defense in depth, and + * MAX_GROUNDEDNESS_PENALTY bounds the deterministic half of it. */ confidenceLevel: ConfidenceLevel; /** Search results used as context for generation */ diff --git a/packages/outpost/shared/src/constants.ts b/packages/outpost/shared/src/constants.ts index ec4e799b..f31b2862 100644 --- a/packages/outpost/shared/src/constants.ts +++ b/packages/outpost/shared/src/constants.ts @@ -28,11 +28,20 @@ export const DEFAULT_SLA_RESOLUTION: Record = { /** * AI confidence thresholds. * - * Two overlapping schemes coexist here: - * - Action-based (AUTO_RESPOND / SUGGEST / ESCALATE): used by the - * response-generation pipeline to decide what action to take. - * - Level-based (HIGH_THRESHOLD / MEDIUM_THRESHOLD): used by the - * dashboard and analytics to bucket responses into confidence tiers. + * Three groups, and only two of them have readers: + * - ESCALATE — live. The pipeline picks the disclaimer on it and the queue + * handler enqueues an ESCALATION job below it. + * - HIGH_THRESHOLD / MEDIUM_THRESHOLD — live. `classifyConfidence` buckets + * responses into tiers for the disclaimer, the dashboard, and analytics. + * - AUTO_RESPOND / SUGGEST — NO READER in `src/`. They described an + * action-based scheme the pipeline never implemented: every response posts, + * and what varies is the disclaimer and whether a human is paged. The last + * reader was `GeneratedResponse.autoSend`, removed because it derived from a + * pre-deduction score and so read `true` for exactly the responses the + * groundedness gate withholds. Kept for the documented band and because + * queue/github-app fixtures still reference the numbers; delete both if a + * genuine auto-post gate is ever built, rather than wiring them to + * something new that happens to want a 0.9 cutoff. */ export const AI_CONFIDENCE = { /** Above this threshold, auto-respond */ From e8f233dd5067031f4613ed6c13bca1365b48cd9a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:36:46 +0000 Subject: [PATCH 44/83] chore(deps): update github actions --- .github/workflows/ci.yml | 2 +- .github/workflows/security_zizmor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc75fce4..451be11b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: persist-credentials: false - name: Setup pnpm - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 diff --git a/.github/workflows/security_zizmor.yml b/.github/workflows/security_zizmor.yml index 26bdaa83..99d80567 100644 --- a/.github/workflows/security_zizmor.yml +++ b/.github/workflows/security_zizmor.yml @@ -38,7 +38,7 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 with: min-severity: medium advanced-security: false From 1a2332201b25c42275012ff5a872f731b852b938 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 5 Aug 2026 07:05:22 -0400 Subject: [PATCH 45/83] =?UTF-8?q?fix(sync):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20typed=20deps,=20corrupt-config=20diagnosis,=20boot=20semanti?= =?UTF-8?q?cs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the three change requests on #95 plus the open question. Typed deps instead of `as never` (build-sync-engine.ts, 5 sites) `as never` silenced the check completely and left no record of the intended contract — worse than the `as any` it replaced, since `never` is assignable to everything and nothing breaks if a contract gains a required member. Each cast now names the contract the PR already exports: StatusMapDb, PriorityMapDb, LabelMapperDb, SyncEngineDeps['prisma'], SyncEngineDeps['createJob'], and IdentityMapperDeps. Same deliberate coercion, but it fails loudly on drift. This carries forward the approach #135 introduced (merged 2026-07-28) into the new module, rather than deleting it. A corrupt config row is no longer indistinguishable from no config (mappings route) `readPersistedConfig` returned null both when the row was absent and when it would not parse, so GET served code defaults and the dashboard rendered them as the saved settings — an admin's configuration appearing to silently revert, with nothing logged. It now returns a discriminated result (absent | corrupt | ok), logs the reason with the config key, and GET reports `configSource` ('persisted' or 'defaults') plus `configError` when the row is unusable. The read path also validates rather than casting. `isValidMappingShape` guarded only PUT, so a row written by an older version of the code — or hand-edited in the database — reached the worker unvalidated. Read now runs the same check against the same `TicketStatus` / `TicketPriority` enum sources the PUT path uses. Boot semantics of the top-level await, documented `await buildSyncEngine()` performs three database reads before the entry module finishes evaluating, so an unreachable database throws during import and the process exits before the health server starts listening — the container crash-loops with no /health rather than coming up degraded. That is the intent (a worker running on silently-defaulted mappings would write wrong statuses to Linear), and the comment now says so, including that #138's /health work makes a degraded-but-listening mode a real option worth revisiting. Conflict resolution (b1bd3a2, kept) Resolved by hand rather than taking either side wholesale, precisely because taking this branch's file would have silently unregistered GITHUB_REACTION_POLL. All four of main's references survive: the header docstring, the import, the concurrencyByType entry, and the worker.on registration. Verified by grep. Tests Four new cases in apps/web/src/__tests__/sync-mappings-config-read.test.ts cover absent, valid, unparseable, and parses-but-wrong-shape. Red-green verified: reverting the corrupt-vs-absent distinction fails the unparseable case. Verified: build 10/10, typecheck 10/10, 1,750 tests pass. CI was already green on the merge commit (b1bd3a2) before these changes. --- .../sync-mappings-config-read.test.ts | 101 ++++++++++++++++++ apps/web/src/app/api/sync/mappings/route.ts | 66 ++++++++++-- apps/worker/src/build-sync-engine.ts | 24 ++++- apps/worker/src/index.ts | 14 +++ 4 files changed, 192 insertions(+), 13 deletions(-) create mode 100644 apps/web/src/__tests__/sync-mappings-config-read.test.ts diff --git a/apps/web/src/__tests__/sync-mappings-config-read.test.ts b/apps/web/src/__tests__/sync-mappings-config-read.test.ts new file mode 100644 index 00000000..efe66322 --- /dev/null +++ b/apps/web/src/__tests__/sync-mappings-config-read.test.ts @@ -0,0 +1,101 @@ +/** + * Read-path behaviour of GET /api/sync/mappings. + * + * The endpoint exists partly because its predecessor accepted edits and threw + * them away. These tests pin that the same failure shape cannot come back on the + * READ side: a SystemConfig row that exists but is unusable must not be reported + * as "never configured", because that renders the code defaults as though they + * were the admin's saved settings. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +const mockSystemConfigFindUnique = vi.fn(); +const mockExternalIdentityFindMany = vi.fn(); + +vi.mock('@copilotkit/outpost/db', () => ({ + prisma: { + systemConfig: { + findUnique: (...args: unknown[]) => mockSystemConfigFindUnique(...args), + }, + externalIdentity: { + findMany: (...args: unknown[]) => mockExternalIdentityFindMany(...args), + }, + }, +})); + +vi.mock('@/lib/require-admin', () => ({ + requireSession: async () => ({ error: null }), + requireAdmin: async () => ({ error: null }), +})); + +// Import after mocks +import { GET } from '@/app/api/sync/mappings/route'; + +const VALID_CONFIG = { + statusMappings: { + linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }], + }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, +}; + +describe('GET /api/sync/mappings — persisted config read path', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockExternalIdentityFindMany.mockResolvedValue([]); + }); + + it('reports defaults as defaults when no row exists', async () => { + mockSystemConfigFindUnique.mockResolvedValue(null); + + const body = await (await GET()).json(); + + expect(body.configSource).toBe('defaults'); + expect(body.configError).toBeUndefined(); + }); + + it('serves the persisted config and says so when the row is valid', async () => { + mockSystemConfigFindUnique.mockResolvedValue({ value: JSON.stringify(VALID_CONFIG) }); + + const body = await (await GET()).json(); + + expect(body.configSource).toBe('persisted'); + expect(body.statusMappings).toEqual(VALID_CONFIG.statusMappings); + }); + + // Previously identical to "absent": the admin's config appeared to revert + // with nothing in the logs. + it('distinguishes an unparseable row from an absent one, and logs it', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockSystemConfigFindUnique.mockResolvedValue({ value: '{not json' }); + + const body = await (await GET()).json(); + + expect(body.configSource).toBe('defaults'); + expect(body.configError).toContain('JSON'); + expect(errorSpy.mock.calls.flat().join(' ')).toContain('sync.mappingConfig'); + errorSpy.mockRestore(); + }); + + // A row written by an older version of the code, or hand-edited in the DB, + // reached the worker unvalidated because the read path only cast. + it('rejects a row that parses but does not match the mapping shape', async () => { + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockSystemConfigFindUnique.mockResolvedValue({ + value: JSON.stringify({ + statusMappings: { + linear: [{ externalStatus: 'Done', outpostStatus: 'NOT_A_STATUS' }], + }, + priorityMappings: VALID_CONFIG.priorityMappings, + }), + }); + + const body = await (await GET()).json(); + + expect(body.configSource).toBe('defaults'); + expect(body.configError).toContain('shape'); + expect(errorSpy).toHaveBeenCalled(); + errorSpy.mockRestore(); + }); +}); diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 335f1487..3911863b 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -68,14 +68,59 @@ interface PersistedMappingConfig { labelRules?: typeof DEFAULT_LABEL_RULES; } -async function readPersistedConfig(): Promise { +/** + * Outcome of reading the persisted mapping config. + * + * `absent` and `corrupt` are deliberately distinct. Collapsing both to null made + * a row that exists but cannot be used indistinguishable from "never configured": + * GET served the code defaults, the dashboard rendered them as if they were the + * saved settings, and an admin's configuration appeared to silently revert with + * nothing in the logs. This endpoint exists partly because the old one accepted + * edits and threw them away — the same failure shape must not return on the read + * path. + */ +type PersistedConfigRead = + | { status: 'absent' } + | { status: 'corrupt'; reason: string } + | { status: 'ok'; config: PersistedMappingConfig }; + +async function readPersistedConfig(): Promise { const row = await prisma.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); - if (!row) return null; + if (!row) return { status: 'absent' }; + + let parsed: unknown; try { - return JSON.parse(row.value) as PersistedMappingConfig; - } catch { - return null; + parsed = JSON.parse(row.value); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + console.error( + `[sync/mappings] SystemConfig row "${MAPPING_CONFIG_KEY}" is not valid JSON — ` + + `serving code defaults and IGNORING the persisted config: ${reason}`, + ); + return { status: 'corrupt', reason: 'row is not valid JSON' }; } + + // The PUT path validates before writing, but a row written by an older + // version of this code — or hand-edited in the database — reaches the worker + // unvalidated. Re-validate on read rather than trusting the cast. + if ( + !isValidMappingShape( + (parsed as PersistedMappingConfig)?.statusMappings, + Object.values(TicketStatus), + ) || + !isValidMappingShape( + (parsed as PersistedMappingConfig)?.priorityMappings, + Object.values(TicketPriority), + ) + ) { + console.error( + `[sync/mappings] SystemConfig row "${MAPPING_CONFIG_KEY}" parsed but does not match the ` + + 'expected mapping shape — serving code defaults and IGNORING the persisted config.', + ); + return { status: 'corrupt', reason: 'row does not match the expected mapping shape' }; + } + + return { status: 'ok', config: parsed as PersistedMappingConfig }; } /** @@ -104,12 +149,17 @@ export async function GET() { })); const persisted = await readPersistedConfig(); + const config = persisted.status === 'ok' ? persisted.config : null; return NextResponse.json({ - statusMappings: persisted?.statusMappings ?? DEFAULT_STATUS_MAPPINGS, - priorityMappings: persisted?.priorityMappings ?? DEFAULT_PRIORITY_MAPPINGS, + statusMappings: config?.statusMappings ?? DEFAULT_STATUS_MAPPINGS, + priorityMappings: config?.priorityMappings ?? DEFAULT_PRIORITY_MAPPINGS, identityMappings, - labelRules: persisted?.labelRules ?? DEFAULT_LABEL_RULES, + labelRules: config?.labelRules ?? DEFAULT_LABEL_RULES, + // Tells the caller these ARE the code defaults and why, so the dashboard + // can say so instead of presenting them as the saved configuration. + configSource: persisted.status === 'ok' ? 'persisted' : 'defaults', + ...(persisted.status === 'corrupt' ? { configError: persisted.reason } : {}), }); } diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index 475c0e19..29d721ce 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -18,20 +18,34 @@ import { loadLabelMapper, initializeSyncEngine, type SyncEngine, + type SyncEngineDeps, + type StatusMapDb, + type PriorityMapDb, + type LabelMapperDb, + type IdentityMapperDeps, } from '@copilotkit/outpost/shared'; export async function buildSyncEngine(): Promise { // Load all three persisted mapping configs (status / priority / label), // each falling back to its hardcoded default when nothing is persisted. + // The casts below are deliberate: each *Db / *Deps type describes only the + // slice of Prisma its consumer needs, with loose Record + // argument shapes, so the real PrismaClient is not assignable in the strict + // direction. Asserting to the NAMED contract rather than `as never` keeps the + // intent legible and — unlike `never`, which is assignable to everything — + // breaks loudly if any of these contracts gains a required member. const [statusMap, priorityMap, labelMapper] = await Promise.all([ - loadStatusMap('linear', prisma as never), - loadPriorityMap('linear', prisma as never), - loadLabelMapper('linear', prisma as never), + loadStatusMap('linear', prisma as unknown as StatusMapDb), + loadPriorityMap('linear', prisma as unknown as PriorityMapDb), + loadLabelMapper('linear', prisma as unknown as LabelMapperDb), ]); return initializeSyncEngine({ - deps: { prisma: prisma as never, createJob: createJob as never }, - identityDeps: prisma as never, + deps: { + prisma: prisma as unknown as SyncEngineDeps['prisma'], + createJob: createJob as SyncEngineDeps['createJob'], + }, + identityDeps: prisma as unknown as IdentityMapperDeps, statusMapOverride: statusMap, priorityMapOverride: priorityMap, labelMapperOverride: labelMapper, diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 45402c15..e33f1019 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -41,6 +41,20 @@ import { buildSyncEngine } from './build-sync-engine.js'; // the prisma/createJob coercion and the plugin registration that used to be // inline here. main's inline version is therefore dropped rather than merged; // handleGithubReactionPoll, added on main, is kept — it is registered below. +// +// BOOT SEMANTICS — deliberate change. This is a top-level await that performs +// three database reads (the persisted status / priority / label mapping configs) +// before this module finishes evaluating. If the database is unreachable at boot +// the import throws, so the process exits BEFORE the health server below starts +// listening: the container crash-loops with no /health at all rather than coming +// up and reporting itself degraded. +// +// Fail-fast is the intent — a worker running with silently-defaulted mappings is +// worse than one that is visibly down, since TRACKER_SYNC would then write wrong +// statuses to Linear. Railway's restart policy is the retry mechanism. Note this +// interacts with the /health honesty follow-up (#138): once /health reflects +// worker state, a degraded-but-listening mode becomes a real option and this +// decision is worth revisiting. const syncEngine = await buildSyncEngine(); const handleTrackerSync = createTrackerSyncHandler(syncEngine); From d57aac885d13979ff35606ba8720699414a35b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:15:50 -0400 Subject: [PATCH 46/83] fix(sync): close the three open items from the 2026-07-18 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 07-18 review approved with five comments; items 4 (`as never`) and 5 (top-level await) were handled in the previous commit. These are 1-3, which were never addressed. 1. Dashboard/worker divergence on empty arrays `isValidMappingShape` rejected an empty OBJECT but accepted `{ linear: [] }` — the inner array passed `.every` vacuously. Meanwhile `loadStatusMap` / `loadPriorityMap` treat `entries.length === 0` as "nothing persisted, use the hardcoded defaults" (status-map.ts:131). So saving `{ linear: [] }` left the dashboard showing NO mappings while the worker kept applying Linear's defaults. Empty per-plugin arrays are now rejected on save, for the same reason the empty-object case already was: a save must not be able to produce a state where the UI and the engine disagree about what is in effect. The read path validates per SECTION rather than all-or-nothing, so a row with unusable priorityMappings no longer discards perfectly good statusMappings. Each bad section falls back on its own, is logged by name, and is reported to the caller via `invalidSections` so the dashboard can mark it as defaults. 2. Sequential enqueue in force-sync `POST /api/sync/force` issued 2xN sequential inserts, a long round-trip chain that could brush the route timeout on a large workspace. Now enqueued via Promise.all. Deliberately Promise.all and NOT allSettled: a failed enqueue must still propagate so the route 500s. I first wrote this as a 207 partial-success response and that was wrong — it contradicted behaviour this review explicitly praised (narrowing the try/catch so a DB failure surfaces as 500 rather than a disguised 400) and pinned by "does not mislabel a mid-loop DB/queue error". Reverted. 3. Force-sync is not atomic Confirmed as a conscious choice and documented rather than changed. If one insert fails, jobs already enqueued stay and a retry re-enqueues from scratch. That is tolerable here because TRACKER_SYNC pushes current ticket state, so a duplicate is a no-op in effect, and this is a manual admin action rather than an automated path. Test fixtures Nine `priorityMappings: { linear: [] }` and six `statusMappings: { linear: [] }` fixtures encoded the state item 1 now forbids; they carry real mappings now. One of them mattered beyond bookkeeping: "rejects labelRules of the wrong type" had an empty statusMappings, so it returned 400 for the wrong reason and would have passed even with labelRules validation deleted. New tests: an empty per-plugin array is rejected on save (red-green verified — removing the check fails it), and the read path keeps the valid section while naming the invalid one. Verified: typecheck 10/10, 1,751 tests pass. --- apps/web/src/__tests__/sync-api.test.ts | 68 +++++++++++++++---- .../sync-mappings-config-read.test.ts | 16 +++-- apps/web/src/app/api/sync/force/route.ts | 52 +++++++++----- apps/web/src/app/api/sync/mappings/route.ts | 56 +++++++++++---- 4 files changed, 141 insertions(+), 51 deletions(-) diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 2b0d6fbe..f68a1ba3 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -280,7 +280,9 @@ describe('GET /api/sync/mappings', () => { mockExternalIdentityFindMany.mockResolvedValue([]); const saved = { statusMappings: { linear: [{ externalStatus: 'Custom', outpostStatus: 'OPEN' }] }, - priorityMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, labelRules: { linear: [] }, }; mockSystemConfigFindUnique.mockResolvedValue({ @@ -323,7 +325,9 @@ describe('PUT /api/sync/mappings', () => { it('persists a valid mapping update', async () => { const config = { statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, - priorityMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, }; mockSystemConfigUpsert.mockResolvedValue({ key: 'sync.mappingConfig', @@ -356,7 +360,9 @@ describe('PUT /api/sync/mappings', () => { 'http://localhost:3000/api/sync/mappings', { statusMappings: 'garbage', - priorityMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, }, 'PUT', ); @@ -366,12 +372,36 @@ describe('PUT /api/sync/mappings', () => { expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); }); + // An empty per-plugin array used to pass validation, but loadStatusMap and + // loadPriorityMap treat an empty array as "nothing persisted" and fall back to + // the hardcoded defaults — so saving this left the dashboard showing NO + // mappings while the worker kept applying Linear's defaults. + it('rejects an empty per-plugin mapping array', async () => { + const req = makeJsonRequest( + 'http://localhost:3000/api/sync/mappings', + { + statusMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, + }, + 'PUT', + ); + + const res = await putMappings(req as never); + + expect(res.status).toBe(400); + expect(mockSystemConfigUpsert).not.toHaveBeenCalled(); + }); + it('rejects a mapping with an unknown outpostStatus value', async () => { const req = makeJsonRequest( 'http://localhost:3000/api/sync/mappings', { statusMappings: { linear: [{ externalStatus: 'X', outpostStatus: 'NOT_REAL' }] }, - priorityMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, }, 'PUT', ); @@ -385,7 +415,7 @@ describe('PUT /api/sync/mappings', () => { const req = makeJsonRequest( 'http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, priorityMappings: { linear: [{ externalPriority: 'X', outpostPriority: 'NOT_REAL' }], }, @@ -404,8 +434,10 @@ describe('PUT /api/sync/mappings', () => { const req = makeJsonRequest( 'http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, }, 'PUT', ); @@ -434,8 +466,10 @@ describe('PUT /api/sync/mappings', () => { const req = makeJsonRequest( 'http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, labelRules: 'garbage', }, 'PUT', @@ -448,8 +482,10 @@ describe('PUT /api/sync/mappings', () => { it('persists a valid labelRules update', async () => { const config = { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, labelRules: { linear: [{ externalPrefix: 'Priority: ', outpostPrefix: '' }] }, }; mockSystemConfigUpsert.mockResolvedValue({ @@ -474,8 +510,10 @@ describe('PUT /api/sync/mappings', () => { const req = makeJsonRequest( 'http://localhost:3000/api/sync/mappings', { - statusMappings: { linear: [] }, - priorityMappings: { linear: [] }, + statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, labelRules: {}, }, 'PUT', @@ -489,7 +527,9 @@ describe('PUT /api/sync/mappings', () => { it('does not mislabel a DB write failure as "Invalid request body"', async () => { const config = { statusMappings: { linear: [{ externalStatus: 'Done', outpostStatus: 'RESOLVED' }] }, - priorityMappings: { linear: [] }, + priorityMappings: { + linear: [{ externalPriority: 'Urgent', outpostPriority: 'CRITICAL' }], + }, }; mockSystemConfigUpsert.mockRejectedValueOnce(new Error('db down')); diff --git a/apps/web/src/__tests__/sync-mappings-config-read.test.ts b/apps/web/src/__tests__/sync-mappings-config-read.test.ts index efe66322..6036580a 100644 --- a/apps/web/src/__tests__/sync-mappings-config-read.test.ts +++ b/apps/web/src/__tests__/sync-mappings-config-read.test.ts @@ -80,7 +80,10 @@ describe('GET /api/sync/mappings — persisted config read path', () => { // A row written by an older version of the code, or hand-edited in the DB, // reached the worker unvalidated because the read path only cast. - it('rejects a row that parses but does not match the mapping shape', async () => { + // + // Validated per section: one unusable section must not discard the other, so + // the good half is still served and the bad half is named. + it('falls back per section, keeping the valid half and naming the bad one', async () => { const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); mockSystemConfigFindUnique.mockResolvedValue({ value: JSON.stringify({ @@ -93,9 +96,14 @@ describe('GET /api/sync/mappings — persisted config read path', () => { const body = await (await GET()).json(); - expect(body.configSource).toBe('defaults'); - expect(body.configError).toContain('shape'); - expect(errorSpy).toHaveBeenCalled(); + expect(body.invalidSections).toEqual(['statusMappings']); + // The bad section serves code defaults... + expect(body.statusMappings).not.toEqual({ + linear: [{ externalStatus: 'Done', outpostStatus: 'NOT_A_STATUS' }], + }); + // ...while the good one is still the admin's saved config. + expect(body.priorityMappings).toEqual(VALID_CONFIG.priorityMappings); + expect(errorSpy.mock.calls.flat().join(' ')).toContain('statusMappings'); errorSpy.mockRestore(); }); }); diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index 02a5801c..b2b56bfd 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -50,24 +50,40 @@ export async function POST(request: NextRequest) { include: { ticket: true }, }); - let jobs = 0; - for (const link of links) { - await createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'status_change', - changeData: { status: link.ticket.status }, - }); - jobs += 1; - - await createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'priority_change', - changeData: { priority: link.ticket.priority }, - }); - jobs += 1; - } + // Enqueued in parallel rather than 2xN sequential round-trips: a plugin with + // many linked tickets made this a long chain that could brush the route + // timeout on a large workspace. + // + // Promise.all, NOT allSettled: a failed enqueue must still propagate so the + // route 500s. That behaviour is deliberate and pinned by + // sync-api.test.ts ("does not mislabel a mid-loop DB/queue error") — a DB or + // queue failure disguised as a 4xx was the bug this endpoint's error handling + // was narrowed to fix. + // + // This operation is therefore NOT atomic, by conscious choice: if one insert + // fails, jobs already enqueued stay enqueued and a retry re-enqueues from + // scratch, producing duplicate TRACKER_SYNC jobs. Acceptable here because the + // handler is idempotent in effect (it pushes current ticket state, so a + // duplicate write is a no-op) and this is a manual admin action, not an + // automated path. + const jobs = ( + await Promise.all( + links.flatMap((link: (typeof links)[number]) => [ + createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status: link.ticket.status }, + }), + createJob(JobType.TRACKER_SYNC, { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority: link.ticket.priority }, + }), + ]), + ) + ).length; return NextResponse.json({ queued: links.length, jobs }); } diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index 3911863b..bcf8d4fd 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -82,7 +82,7 @@ interface PersistedMappingConfig { type PersistedConfigRead = | { status: 'absent' } | { status: 'corrupt'; reason: string } - | { status: 'ok'; config: PersistedMappingConfig }; + | { status: 'ok'; config: PersistedMappingConfig; invalidSections: string[] }; async function readPersistedConfig(): Promise { const row = await prisma.systemConfig.findUnique({ where: { key: MAPPING_CONFIG_KEY } }); @@ -103,24 +103,36 @@ async function readPersistedConfig(): Promise { // The PUT path validates before writing, but a row written by an older // version of this code — or hand-edited in the database — reaches the worker // unvalidated. Re-validate on read rather than trusting the cast. - if ( - !isValidMappingShape( - (parsed as PersistedMappingConfig)?.statusMappings, - Object.values(TicketStatus), - ) || - !isValidMappingShape( - (parsed as PersistedMappingConfig)?.priorityMappings, - Object.values(TicketPriority), - ) - ) { + // + // Validated PER SECTION: a row whose priorityMappings are unusable should not + // discard perfectly good statusMappings. Each bad section falls back to the + // code defaults on its own and says which one it was. + const candidate = parsed as PersistedMappingConfig; + const bad: string[] = []; + + if (!isValidMappingShape(candidate?.statusMappings, Object.values(TicketStatus))) { + bad.push('statusMappings'); + } + if (!isValidMappingShape(candidate?.priorityMappings, Object.values(TicketPriority))) { + bad.push('priorityMappings'); + } + + if (bad.length > 0) { console.error( - `[sync/mappings] SystemConfig row "${MAPPING_CONFIG_KEY}" parsed but does not match the ` + - 'expected mapping shape — serving code defaults and IGNORING the persisted config.', + `[sync/mappings] SystemConfig row "${MAPPING_CONFIG_KEY}" has unusable section(s): ` + + `${bad.join(', ')} — serving code defaults for those and keeping the rest.`, ); - return { status: 'corrupt', reason: 'row does not match the expected mapping shape' }; } - return { status: 'ok', config: parsed as PersistedMappingConfig }; + return { + status: 'ok', + config: { + ...candidate, + ...(bad.includes('statusMappings') ? { statusMappings: undefined } : {}), + ...(bad.includes('priorityMappings') ? { priorityMappings: undefined } : {}), + } as PersistedMappingConfig, + invalidSections: bad, + }; } /** @@ -160,6 +172,11 @@ export async function GET() { // can say so instead of presenting them as the saved configuration. configSource: persisted.status === 'ok' ? 'persisted' : 'defaults', ...(persisted.status === 'corrupt' ? { configError: persisted.reason } : {}), + // Names the sections that fell back, so the dashboard can mark those as + // defaults instead of presenting them as saved settings. + ...(persisted.status === 'ok' && persisted.invalidSections.length > 0 + ? { invalidSections: persisted.invalidSections } + : {}), }); } @@ -181,6 +198,15 @@ function isValidMappingShape(value: unknown, validOutpostValues: string[]): bool return Object.values(value as Record).every((entries) => { if (!Array.isArray(entries)) return false; + // An empty per-plugin array is rejected, not accepted-as-vacuously-valid. + // `loadStatusMap` / `loadPriorityMap` treat `entries.length === 0` as + // "nothing persisted, use the hardcoded defaults" (status-map.ts:131), so + // saving `{ linear: [] }` would leave the dashboard showing NO mappings + // while the worker kept applying the Linear defaults. Same reason the + // empty-object case above is rejected: a save must not be able to produce + // a state where the UI and the engine disagree about what is in effect. + if (entries.length === 0) return false; + return entries.every((entry) => { if (typeof entry !== 'object' || entry === null) return false; const record = entry as Record; From ad4658ee424ae98eb76ce8471ed9bef2deb2a18f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Thu, 6 Aug 2026 08:45:08 -0400 Subject: [PATCH 47/83] fix(sync): fix the priority-key blocker and the rest of the d57aac8 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BLOCKER — saving the dashboard defaults broke all inbound Linear priority mapping `DEFAULT_PRIORITY_MAPPINGS` served '0 (None)'..'4 (Low)' as the persisted KEYS, but LinearAdapter maps with `String(data.priority ?? '0')` -> '0'..'4' and `createLinearPriorityMap()` is keyed '0'..'4'. The dashboard PUTs back whatever GET served, so one save from the page persisted a PriorityMap no inbound webhook could ever match: every Linear priority fell through to MEDIUM from then on, permanently, with nothing logged. Keys are now the raw values the adapter sends, and the human text moved to a display-only `label` field the loaders ignore. DEFAULT_STATUS_MAPPINGS already matched its factory exactly, which is why priority was the odd one out and did not show up on a read-through. Round-trip test — the structural fix apps/web/src/__tests__/sync-mappings-roundtrip.test.ts GETs the defaults, PUTs them back, then loads the result through the same loadStatusMap/loadPriorityMap/ loadLabelMapper the worker uses and asserts equivalence with the factories. Red-green verified against BOTH bugs independently: restoring '1 (Urgent)' fails it, and dropping the exclude carry-over fails it. loadLabelMapper no longer drops `exclude` Rebuilding from persisted rules returned `new LabelMapper({ rules })`, silently disabling GitHub's wontfix/duplicate/invalid exclusions on the first save. It now inherits the factory's list via a new `getExcludeList()` accessor. Writer and readers now validate the same thing `isValidMappingShape` picked its key pair via `'externalStatus' in record`, so a priority-shaped row inside statusMappings passed PUT and was then discarded by loadStatusMap — accept-then-discard, one layer down from the bug this PR fixes. It now takes the expected key pair explicitly, and rejects blank keys that the loaders' truthiness checks would drop. Two regressions from d57aac8, mine - `configSource` lied when the whole row was unusable: `status: 'ok'` whenever the JSON parsed meant a row with every section invalid reported 'persisted' with no configError — the exact "defaults presented as saved settings" case the field exists to prevent. It now derives from whether any section survived, and the result-type doc comment describes the code again. - `{...candidate, statusMappings: undefined} as PersistedMappingConfig` asserted over a non-optional field. Those fields are optional now and the strip uses `delete` rather than a cast that made the type false. Bounded enqueue instead of unbounded An unbounded `Promise.all` over 2N inserts traded a route timeout for Prisma pool-acquisition timeouts — same outage, less obvious error. Now chunks of 50, still `Promise.all` within a chunk so a failure propagates as a 500 (pinned by "does not mislabel a mid-loop DB/queue error"), still documented as non-atomic by choice. configSource / configError / invalidSections are now read They were computed and consumed by nothing, so the page still presented defaults as saved settings. The mappings page surfaces both a "showing built-in defaults" notice and a per-section "using defaults for X" notice. One read, three loaders buildSyncEngine fired three `systemConfig.findUnique` calls for the same row on every boot. A request-scoped read-once facade collapses it to one, leaving the loaders' signatures and independent fallbacks untouched. Pinned by a test. Silent non-registration now logs `initializeSyncEngine` skipped Linear registration when `identityDeps` was absent even with both env vars set — the same invisible non-registration this PR was written to fix. It logs loudly now. Also corrected `InitOptions.deps`, which was doc-commented "testing" despite being required in production. Smaller Dropped the merge archaeology from apps/worker/src/index.ts, keeping the boot-semantics paragraph. Force route selects only id/status/priority instead of whole ticket rows (assertion updated to match). Test-fixture note: the build-sync-engine test asserted the loaders were called with `prisma` by identity; they now receive the read-once facade, so it asserts the contract (plugin + a usable systemConfig.findUnique) plus the single-read behaviour instead. Verified: build 10/10, typecheck 10/10, 1,756 tests pass. Deferred, with reasoning: exporting MAPPING_CONFIG_KEY and a shared parseMappingConfig() from packages/outpost/shared/src/sync/ (declared in four files today). It is the right consolidation and would subsume the validation divergence, but it touches all four call sites and their tests — better as its own PR than bolted onto this one. --- apps/web/src/__tests__/sync-api.test.ts | 4 +- .../__tests__/sync-mappings-roundtrip.test.ts | 127 +++++++++++++++ apps/web/src/app/api/sync/force/route.ts | 63 +++++--- apps/web/src/app/api/sync/mappings/route.ts | 148 +++++++++++++----- apps/web/src/app/sync/mappings/page.tsx | 44 +++++- .../src/__tests__/build-sync-engine.test.ts | 54 ++++++- apps/worker/src/build-sync-engine.ts | 36 ++++- apps/worker/src/index.ts | 5 - packages/outpost/shared/src/sync/init.ts | 18 ++- packages/outpost/shared/src/sync/label-map.ts | 33 ++-- 10 files changed, 442 insertions(+), 90 deletions(-) create mode 100644 apps/web/src/__tests__/sync-mappings-roundtrip.test.ts diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index f68a1ba3..027b36fb 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -616,7 +616,9 @@ describe('POST /api/sync/force', () => { expect(body).toEqual({ queued: 1, jobs: 2 }); expect(mockTicketExternalLinkFindMany).toHaveBeenCalledWith({ where: { plugin: 'linear', ticketId: 't-1' }, - include: { ticket: true }, + // Narrowed from `include: { ticket: true }`: the route only reads + // id/status/priority, so it no longer pulls whole ticket rows. + select: { ticket: { select: { id: true, status: true, priority: true } } }, }); }); diff --git a/apps/web/src/__tests__/sync-mappings-roundtrip.test.ts b/apps/web/src/__tests__/sync-mappings-roundtrip.test.ts new file mode 100644 index 00000000..8515bd6b --- /dev/null +++ b/apps/web/src/__tests__/sync-mappings-roundtrip.test.ts @@ -0,0 +1,127 @@ +/** + * Round-trip: GET the dashboard defaults, PUT them straight back, then load them + * through the same functions the worker uses. + * + * This is the structural guard against a whole class of bug. The dashboard PUTs + * back exactly what GET served, so any value in `externalStatus`/`externalPriority` + * that is cosmetic rather than the adapter's real lookup key becomes a persisted + * key that can never match — and the failure is silent, because the loaders fall + * back to defaults and every inbound value lands on the fallback priority. + * + * That is exactly what shipped: the Linear priority defaults read '0 (None)'.. + * '4 (Low)' while LinearAdapter maps with String(data.priority) -> '0'..'4'. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { TicketPriority, TicketStatus } from '@copilotkit/outpost/shared'; +import { + loadStatusMap, + loadPriorityMap, + loadLabelMapper, + createLinearStatusMap, + createLinearPriorityMap, + createGitHubLabelMapper, +} from '@copilotkit/outpost/shared'; + +const mockSystemConfigFindUnique = vi.fn(); +const mockSystemConfigUpsert = vi.fn(); +const mockExternalIdentityFindMany = vi.fn(); +const mockGetServerSession = vi.fn(); + +vi.mock('@copilotkit/outpost/db', () => ({ + prisma: { + systemConfig: { + findUnique: (...a: unknown[]) => mockSystemConfigFindUnique(...a), + upsert: (...a: unknown[]) => mockSystemConfigUpsert(...a), + }, + externalIdentity: { findMany: (...a: unknown[]) => mockExternalIdentityFindMany(...a) }, + }, +})); + +vi.mock('@/lib/require-admin', () => ({ + requireSession: async () => ({ error: null }), + requireAdmin: async () => ({ error: null }), +})); + +vi.mock('next-auth', () => ({ getServerSession: (...a: unknown[]) => mockGetServerSession(...a) })); + +import { GET, PUT } from '@/app/api/sync/mappings/route'; + +/** The db shape the loaders need, backed by whatever the PUT persisted. */ +function dbServing(value: string) { + return { systemConfig: { findUnique: async () => ({ key: 'sync.mappingConfig', value }) } }; +} + +describe('mappings round-trip: GET defaults -> PUT -> load as the worker does', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockExternalIdentityFindMany.mockResolvedValue([]); + mockSystemConfigFindUnique.mockResolvedValue(null); // GET serves code defaults + mockSystemConfigUpsert.mockImplementation(async (args: { create: { value: string } }) => ({ + key: 'sync.mappingConfig', + value: args.create.value, + })); + }); + + async function roundTrip() { + const served = await (await GET()).json(); + + const res = await PUT( + new Request('http://localhost:3000/api/sync/mappings', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + statusMappings: served.statusMappings, + priorityMappings: served.priorityMappings, + labelRules: served.labelRules, + }), + }) as never, + ); + expect(res.status).toBe(200); + + const persistedValue = mockSystemConfigUpsert.mock.calls[0][0].create.value as string; + return { served, db: dbServing(persistedValue) }; + } + + it('accepts its own defaults', async () => { + await roundTrip(); + }); + + // The blocker: a saved priority map must still match what LinearAdapter sends. + it('produces a Linear priority map equivalent to the factory default', async () => { + const { db } = await roundTrip(); + + const loaded = await loadPriorityMap('linear', db as never); + const factory = createLinearPriorityMap(); + + for (const raw of ['0', '1', '2', '3', '4']) { + expect(loaded.toOutpost(raw)).toBe(factory.toOutpost(raw)); + } + // And specifically not everything collapsing onto the fallback. + expect(loaded.toOutpost('1')).toBe(TicketPriority.CRITICAL); + expect(loaded.toOutpost('4')).toBe(TicketPriority.LOW); + }); + + it('produces a Linear status map equivalent to the factory default', async () => { + const { db } = await roundTrip(); + + const loaded = await loadStatusMap('linear', db as never); + const factory = createLinearStatusMap(); + + for (const state of ['Triage', 'Backlog', 'Todo', 'In Progress', 'Done', 'Canceled']) { + expect(loaded.toOutpost(state)).toBe(factory.toOutpost(state)); + } + expect(loaded.toOutpost('Done')).toBe(TicketStatus.RESOLVED); + }); + + // The exclude drop: rebuilding from persisted rules must not lose the + // factory's wontfix/duplicate/invalid filtering. + it('keeps GitHub label exclusions after a save', async () => { + const { db } = await roundTrip(); + + const loaded = await loadLabelMapper('github', db as never); + const factory = createGitHubLabelMapper(); + + expect(loaded.toOutpost(['wontfix', 'bug'])).toEqual(factory.toOutpost(['wontfix', 'bug'])); + expect(loaded.toOutpost(['wontfix'])).toEqual([]); + }); +}); diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index b2b56bfd..694e8f63 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -47,15 +47,17 @@ export async function POST(request: NextRequest) { const links = await prisma.ticketExternalLink.findMany({ where: ticketId ? { plugin, ticketId } : { plugin }, - include: { ticket: true }, + // Only the three columns the payloads use, not whole ticket rows. + select: { ticket: { select: { id: true, status: true, priority: true } } }, }); - // Enqueued in parallel rather than 2xN sequential round-trips: a plugin with - // many linked tickets made this a long chain that could brush the route - // timeout on a large workspace. + // Enqueued in BOUNDED batches. Sequential 2xN round-trips could brush the + // route timeout on a large workspace; an unbounded Promise.all over 2N inserts + // just trades that for Prisma pool-acquisition timeouts, which is the same + // outage with a less obvious error. Chunks keep both bounded. // - // Promise.all, NOT allSettled: a failed enqueue must still propagate so the - // route 500s. That behaviour is deliberate and pinned by + // Promise.all within each chunk, NOT allSettled: a failed enqueue must still + // propagate so the route 500s. That is deliberate and pinned by // sync-api.test.ts ("does not mislabel a mid-loop DB/queue error") — a DB or // queue failure disguised as a 4xx was the bug this endpoint's error handling // was narrowed to fix. @@ -63,27 +65,36 @@ export async function POST(request: NextRequest) { // This operation is therefore NOT atomic, by conscious choice: if one insert // fails, jobs already enqueued stay enqueued and a retry re-enqueues from // scratch, producing duplicate TRACKER_SYNC jobs. Acceptable here because the - // handler is idempotent in effect (it pushes current ticket state, so a - // duplicate write is a no-op) and this is a manual admin action, not an - // automated path. - const jobs = ( + // handler pushes current ticket state, so a duplicate write is a no-op in + // effect, and this is a manual admin action rather than an automated path. + const ENQUEUE_CHUNK_SIZE = 50; + + const payloads = links.flatMap((link: (typeof links)[number]) => [ + { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status: link.ticket.status }, + }, + { + ticketId: link.ticket.id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority: link.ticket.priority }, + }, + ]); + + for (let i = 0; i < payloads.length; i += ENQUEUE_CHUNK_SIZE) { await Promise.all( - links.flatMap((link: (typeof links)[number]) => [ - createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'status_change', - changeData: { status: link.ticket.status }, - }), - createJob(JobType.TRACKER_SYNC, { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'priority_change', - changeData: { priority: link.ticket.priority }, - }), - ]), - ) - ).length; + payloads + .slice(i, i + ENQUEUE_CHUNK_SIZE) + .map((payload) => createJob(JobType.TRACKER_SYNC, payload)), + ); + } + + // Every payload either enqueued or the loop above threw, so this is exact + // rather than a count accumulated as we went. + const jobs = payloads.length; return NextResponse.json({ queued: links.length, jobs }); } diff --git a/apps/web/src/app/api/sync/mappings/route.ts b/apps/web/src/app/api/sync/mappings/route.ts index bcf8d4fd..dc26f384 100644 --- a/apps/web/src/app/api/sync/mappings/route.ts +++ b/apps/web/src/app/api/sync/mappings/route.ts @@ -26,16 +26,33 @@ const DEFAULT_STATUS_MAPPINGS: Record< ], }; +/** + * Priority mapping defaults. + * + * `externalPriority` MUST be the value the adapter actually receives, because the + * dashboard PUTs back whatever GET served — so anything cosmetic in this key gets + * persisted as a real lookup key and silently stops matching. + * + * For Linear that is the raw numeric priority: `LinearAdapter` maps with + * `String(data.priority ?? '0')` (adapters/linear.ts:138) and + * `createLinearPriorityMap()` is keyed '0'-'4' (priority-map.ts:71). These keys + * previously read '0 (None)'-'4 (Low)', which no inbound webhook could ever + * match: one save from the dashboard persisted a PriorityMap that matched nothing + * and every inbound Linear priority silently fell through to MEDIUM. + * + * Human-readable text belongs in `label`, which is display-only — the editor + * renders it and the loaders ignore it. + */ const DEFAULT_PRIORITY_MAPPINGS: Record< string, - Array<{ externalPriority: string; outpostPriority: string }> + Array<{ externalPriority: string; outpostPriority: string; label?: string }> > = { linear: [ - { externalPriority: '0 (None)', outpostPriority: 'MEDIUM' }, - { externalPriority: '1 (Urgent)', outpostPriority: 'CRITICAL' }, - { externalPriority: '2 (High)', outpostPriority: 'HIGH' }, - { externalPriority: '3 (Medium)', outpostPriority: 'MEDIUM' }, - { externalPriority: '4 (Low)', outpostPriority: 'LOW' }, + { externalPriority: '0', outpostPriority: 'MEDIUM', label: 'None' }, + { externalPriority: '1', outpostPriority: 'CRITICAL', label: 'Urgent' }, + { externalPriority: '2', outpostPriority: 'HIGH', label: 'High' }, + { externalPriority: '3', outpostPriority: 'MEDIUM', label: 'Medium' }, + { externalPriority: '4', outpostPriority: 'LOW', label: 'Low' }, ], github: [ { externalPriority: 'critical', outpostPriority: 'CRITICAL' }, @@ -63,16 +80,26 @@ const DEFAULT_LABEL_RULES: Record< const MAPPING_CONFIG_KEY = 'sync.mappingConfig'; interface PersistedMappingConfig { - statusMappings: typeof DEFAULT_STATUS_MAPPINGS; - priorityMappings: typeof DEFAULT_PRIORITY_MAPPINGS; + // Optional because the read path strips any section that fails validation and + // lets the caller fall back per section. Previously these were non-optional + // and the strip was a type assertion over a field the type said was always + // present — it worked only because every reader used `?? DEFAULT`. + statusMappings?: typeof DEFAULT_STATUS_MAPPINGS; + priorityMappings?: typeof DEFAULT_PRIORITY_MAPPINGS; labelRules?: typeof DEFAULT_LABEL_RULES; } /** * Outcome of reading the persisted mapping config. * - * `absent` and `corrupt` are deliberately distinct. Collapsing both to null made - * a row that exists but cannot be used indistinguishable from "never configured": + * `absent` and `corrupt` are deliberately distinct. `corrupt` covers a row whose + * JSON does not parse; a row that parses but whose SECTIONS fail validation comes + * back as `ok` with those sections stripped and named in `invalidSections`, so the + * good half survives. Callers decide "am I showing saved settings or defaults?" + * from whether anything survived, not from `status` alone. + * + * Collapsing everything to null made a row that exists but cannot be used + * indistinguishable from "never configured": * GET served the code defaults, the dashboard rendered them as if they were the * saved settings, and an admin's configuration appeared to silently revert with * nothing in the logs. This endpoint exists partly because the old one accepted @@ -110,10 +137,24 @@ async function readPersistedConfig(): Promise { const candidate = parsed as PersistedMappingConfig; const bad: string[] = []; - if (!isValidMappingShape(candidate?.statusMappings, Object.values(TicketStatus))) { + if ( + !isValidMappingShape( + candidate?.statusMappings, + Object.values(TicketStatus), + 'externalStatus', + 'outpostStatus', + ) + ) { bad.push('statusMappings'); } - if (!isValidMappingShape(candidate?.priorityMappings, Object.values(TicketPriority))) { + if ( + !isValidMappingShape( + candidate?.priorityMappings, + Object.values(TicketPriority), + 'externalPriority', + 'outpostPriority', + ) + ) { bad.push('priorityMappings'); } @@ -124,15 +165,11 @@ async function readPersistedConfig(): Promise { ); } - return { - status: 'ok', - config: { - ...candidate, - ...(bad.includes('statusMappings') ? { statusMappings: undefined } : {}), - ...(bad.includes('priorityMappings') ? { priorityMappings: undefined } : {}), - } as PersistedMappingConfig, - invalidSections: bad, - }; + const usable: PersistedMappingConfig = { ...candidate }; + if (bad.includes('statusMappings')) delete usable.statusMappings; + if (bad.includes('priorityMappings')) delete usable.priorityMappings; + + return { status: 'ok', config: usable, invalidSections: bad }; } /** @@ -162,21 +199,29 @@ export async function GET() { const persisted = await readPersistedConfig(); const config = persisted.status === 'ok' ? persisted.config : null; + const invalidSections = persisted.status === 'ok' ? persisted.invalidSections : []; + + // 'persisted' means at least one saved section survived validation. Deriving + // it from `status === 'ok'` alone was wrong: a row whose JSON parsed but whose + // every section failed reported 'persisted' with no configError, which is + // precisely the "defaults presented as saved settings" case this field exists + // to prevent. + const anythingPersisted = + persisted.status === 'ok' && + (config?.statusMappings !== undefined || + config?.priorityMappings !== undefined || + config?.labelRules !== undefined); return NextResponse.json({ statusMappings: config?.statusMappings ?? DEFAULT_STATUS_MAPPINGS, priorityMappings: config?.priorityMappings ?? DEFAULT_PRIORITY_MAPPINGS, identityMappings, labelRules: config?.labelRules ?? DEFAULT_LABEL_RULES, - // Tells the caller these ARE the code defaults and why, so the dashboard - // can say so instead of presenting them as the saved configuration. - configSource: persisted.status === 'ok' ? 'persisted' : 'defaults', + configSource: anythingPersisted ? 'persisted' : 'defaults', ...(persisted.status === 'corrupt' ? { configError: persisted.reason } : {}), - // Names the sections that fell back, so the dashboard can mark those as - // defaults instead of presenting them as saved settings. - ...(persisted.status === 'ok' && persisted.invalidSections.length > 0 - ? { invalidSections: persisted.invalidSections } - : {}), + // Names the sections serving code defaults, so the dashboard can mark + // those rather than presenting them as saved settings. + ...(invalidSections.length > 0 ? { invalidSections } : {}), }); } @@ -186,7 +231,12 @@ export async function GET() { * `{ externalStatus: string; outpostStatus: }` * (or the priority equivalent, keyed `externalPriority`/`outpostPriority`). */ -function isValidMappingShape(value: unknown, validOutpostValues: string[]): boolean { +function isValidMappingShape( + value: unknown, + validOutpostValues: string[], + externalKey: 'externalStatus' | 'externalPriority' = 'externalStatus', + outpostKey: 'outpostStatus' | 'outpostPriority' = 'outpostStatus', +): boolean { if (typeof value !== 'object' || value === null || Array.isArray(value)) { return false; } @@ -210,13 +260,23 @@ function isValidMappingShape(value: unknown, validOutpostValues: string[]): bool return entries.every((entry) => { if (typeof entry !== 'object' || entry === null) return false; const record = entry as Record; - const externalKey = 'externalStatus' in record ? 'externalStatus' : 'externalPriority'; - const outpostKey = 'externalStatus' in record ? 'outpostStatus' : 'outpostPriority'; + + // Validate against the key pair this SECTION requires, not whichever + // key the entry happens to carry. Picking the pair from the entry let + // a priority-shaped row sit inside statusMappings and pass PUT, only + // for loadStatusMap to drop it for having no externalStatus — the + // accept-then-discard behaviour this endpoint exists to remove, moved + // one layer down. + const external = record[externalKey]; + const outpost = record[outpostKey]; return ( - typeof record[externalKey] === 'string' && - typeof record[outpostKey] === 'string' && - validOutpostValues.includes(record[outpostKey] as string) + typeof external === 'string' && + // Non-blank: the loaders test truthiness, so '' would pass here + // and then be discarded there. + external.trim().length > 0 && + typeof outpost === 'string' && + validOutpostValues.includes(outpost) ); }); }); @@ -277,14 +337,28 @@ export async function PUT(request: NextRequest) { ); } - if (!isValidMappingShape(body.statusMappings, Object.values(TicketStatus))) { + if ( + !isValidMappingShape( + body.statusMappings, + Object.values(TicketStatus), + 'externalStatus', + 'outpostStatus', + ) + ) { return NextResponse.json( { error: 'statusMappings has invalid shape or unknown outpostStatus value' }, { status: 400 }, ); } - if (!isValidMappingShape(body.priorityMappings, Object.values(TicketPriority))) { + if ( + !isValidMappingShape( + body.priorityMappings, + Object.values(TicketPriority), + 'externalPriority', + 'outpostPriority', + ) + ) { return NextResponse.json( { error: 'priorityMappings has invalid shape or unknown outpostPriority value' }, { status: 400 }, diff --git a/apps/web/src/app/sync/mappings/page.tsx b/apps/web/src/app/sync/mappings/page.tsx index 25b0a4d2..14932e92 100644 --- a/apps/web/src/app/sync/mappings/page.tsx +++ b/apps/web/src/app/sync/mappings/page.tsx @@ -6,8 +6,16 @@ import { PageHeader } from '@/components/page-header'; import { MappingEditor } from '@/components/sync/mapping-editor'; import type { MappingConfig } from '@/lib/mock-sync'; +/** Provenance the mappings API reports alongside the config it serves. */ +interface ConfigProvenance { + configSource?: 'persisted' | 'defaults'; + configError?: string; + invalidSections?: string[]; +} + export default function MappingsPage() { const [config, setConfig] = useState(null); + const [provenance, setProvenance] = useState({}); const [saving, setSaving] = useState(false); const [saveMessage, setSaveMessage] = useState(null); @@ -17,6 +25,11 @@ export default function MappingsPage() { const res = await fetch('/api/sync/mappings'); const data = await res.json(); setConfig(data); + setProvenance({ + configSource: data.configSource, + configError: data.configError, + invalidSections: data.invalidSections, + }); } catch { // noop } @@ -53,12 +66,35 @@ export default function MappingsPage() { title="Mapping Configuration" description="Configure how statuses, priorities, identities, and labels map between systems." icon={Settings2} - breadcrumbs={[ - { label: 'Sync', href: '/sync' }, - { label: 'Mappings' }, - ]} + breadcrumbs={[{ label: 'Sync', href: '/sync' }, { label: 'Mappings' }]} /> + {/* + * Say when what is on screen is NOT the saved configuration. The API + * reports this, and without surfacing it the page renders code + * defaults identically to persisted settings — so an admin whose row + * is unusable sees no difference and re-saves the defaults over it. + */} + {provenance.configSource === 'defaults' && ( +
+ Showing built-in defaults — no saved mapping configuration is in effect. + {provenance.configError ? ` (${provenance.configError})` : ''} +
+ )} + + {provenance.invalidSections && provenance.invalidSections.length > 0 && ( +
+ Using built-in defaults for {provenance.invalidSections.join(', ')} — the saved + values could not be read. Saving will overwrite them. +
+ )} + {saveMessage && (
({ initializeSyncEngine: (...args: unknown[]) => mockInitializeSyncEngine(...args), })); +const mockConfigFindUnique = vi.fn(); + vi.mock('@copilotkit/outpost/db', () => ({ - prisma: { systemConfig: {}, externalIdentity: {} }, + prisma: { + systemConfig: { findUnique: (...args: unknown[]) => mockConfigFindUnique(...args) }, + externalIdentity: {}, + }, })); vi.mock('@copilotkit/outpost/queue', () => ({ @@ -41,9 +46,18 @@ describe('buildSyncEngine', () => { const result = await buildSyncEngine(); - expect(mockLoadStatusMap).toHaveBeenCalledWith('linear', prisma); - expect(mockLoadPriorityMap).toHaveBeenCalledWith('linear', prisma); - expect(mockLoadLabelMapper).toHaveBeenCalledWith('linear', prisma); + // The loaders each receive a db, not `prisma` itself: they now share a + // read-once facade over it (see singleReadConfigDb). Assert the contract — + // plugin name plus a usable systemConfig.findUnique — rather than object + // identity, which was only ever incidental. + for (const loader of [mockLoadStatusMap, mockLoadPriorityMap, mockLoadLabelMapper]) { + expect(loader).toHaveBeenCalledTimes(1); + const [plugin, db] = loader.mock.calls[0]; + expect(plugin).toBe('linear'); + expect( + typeof (db as { systemConfig: { findUnique: unknown } }).systemConfig.findUnique, + ).toBe('function'); + } expect(mockInitializeSyncEngine).toHaveBeenCalledWith({ deps: { prisma, createJob }, identityDeps: prisma, @@ -53,4 +67,36 @@ describe('buildSyncEngine', () => { }); expect(result).toBe(fakeEngine); }); + + // All three loaders read the same SystemConfig row, which meant three + // identical queries on every worker boot. + it('reads the mapping config row once, however many loaders ask for it', async () => { + mockConfigFindUnique.mockResolvedValue({ key: 'sync.mappingConfig', value: '{}' }); + mockInitializeSyncEngine.mockReturnValue({ getPlugin: vi.fn() }); + + // Real loaders would call through; here the mocked ones do not, so drive + // the facade directly with the db each loader was handed. + mockLoadStatusMap.mockImplementation( + async ( + _plugin: string, + db: { systemConfig: { findUnique: (a: unknown) => unknown } }, + ) => db.systemConfig.findUnique({ where: { key: 'sync.mappingConfig' } }), + ); + mockLoadPriorityMap.mockImplementation( + async ( + _plugin: string, + db: { systemConfig: { findUnique: (a: unknown) => unknown } }, + ) => db.systemConfig.findUnique({ where: { key: 'sync.mappingConfig' } }), + ); + mockLoadLabelMapper.mockImplementation( + async ( + _plugin: string, + db: { systemConfig: { findUnique: (a: unknown) => unknown } }, + ) => db.systemConfig.findUnique({ where: { key: 'sync.mappingConfig' } }), + ); + + await buildSyncEngine(); + + expect(mockConfigFindUnique).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index 29d721ce..a3374a82 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -25,6 +25,29 @@ import { type IdentityMapperDeps, } from '@copilotkit/outpost/shared'; +/** + * Wraps a db so repeated `systemConfig.findUnique` calls for the same key share + * one round-trip. Scoped to a single buildSyncEngine() call, so there is no + * staleness window — the cache dies with the function. + */ +function singleReadConfigDb(db: StatusMapDb): StatusMapDb { + const inFlight = new Map>(); + + return { + systemConfig: { + findUnique: (args: { where: { key: string } }) => { + const key = args.where.key; + let promise = inFlight.get(key); + if (!promise) { + promise = db.systemConfig.findUnique(args); + inFlight.set(key, promise); + } + return promise; + }, + }, + } as unknown as StatusMapDb; +} + export async function buildSyncEngine(): Promise { // Load all three persisted mapping configs (status / priority / label), // each falling back to its hardcoded default when nothing is persisted. @@ -34,10 +57,17 @@ export async function buildSyncEngine(): Promise { // direction. Asserting to the NAMED contract rather than `as never` keeps the // intent legible and — unlike `never`, which is assignable to everything — // breaks loudly if any of these contracts gains a required member. + // One read, three builders. Each loader takes a db and does its own + // findUnique, so calling all three fetched the SAME SystemConfig row three + // times on every worker boot. A tiny in-memory cache in front of them keeps + // the loaders' signatures (and their independent fallbacks) unchanged while + // collapsing it to a single query. + const cachedDb = singleReadConfigDb(prisma as unknown as StatusMapDb); + const [statusMap, priorityMap, labelMapper] = await Promise.all([ - loadStatusMap('linear', prisma as unknown as StatusMapDb), - loadPriorityMap('linear', prisma as unknown as PriorityMapDb), - loadLabelMapper('linear', prisma as unknown as LabelMapperDb), + loadStatusMap('linear', cachedDb as unknown as StatusMapDb), + loadPriorityMap('linear', cachedDb as unknown as PriorityMapDb), + loadLabelMapper('linear', cachedDb as unknown as LabelMapperDb), ]); return initializeSyncEngine({ diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index e33f1019..13d2b9a3 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -37,11 +37,6 @@ import { buildSyncEngine } from './build-sync-engine.js'; // ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── -// This PR moves the engine's construction into build-sync-engine.ts, which owns -// the prisma/createJob coercion and the plugin registration that used to be -// inline here. main's inline version is therefore dropped rather than merged; -// handleGithubReactionPoll, added on main, is kept — it is registered below. -// // BOOT SEMANTICS — deliberate change. This is a top-level await that performs // three database reads (the persisted status / priority / label mapping configs) // before this module finishes evaluating. If the database is unreachable at boot diff --git a/packages/outpost/shared/src/sync/init.ts b/packages/outpost/shared/src/sync/init.ts index 74052436..fceb16bb 100644 --- a/packages/outpost/shared/src/sync/init.ts +++ b/packages/outpost/shared/src/sync/init.ts @@ -18,7 +18,10 @@ import type { IdentityMapperDeps } from './identity-map.js'; // ─── Configuration ────────────────────────────────────────────────────── interface InitOptions { - /** Override for dependency injection (testing). */ + /** + * Prisma + createJob the engine runs against. REQUIRED — despite the wording + * this replaces, it is not a testing-only override; production passes it too. + */ deps: SyncEngineDeps; /** Override for identity mapper deps (testing). */ identityDeps?: IdentityMapperDeps; @@ -56,6 +59,19 @@ export function initializeSyncEngine(options: InitOptions): SyncEngine { ? new IdentityMapper(options.identityDeps) : null; + if (!identityMapper) { + // Both env vars are set, so the operator intended Linear sync — but + // without identityDeps no adapter is registered and TRACKER_SYNC jobs + // fail with "Plugin is not registered". Silence here reproduces the + // exact invisible non-registration this function was extracted to fix, + // so it is loud instead. + console.error( + '[SyncEngine] LINEAR_API_KEY and LINEAR_TEAM_ID are set but identityDeps was not ' + + 'provided — the Linear adapter is NOT registered and outbound Linear sync will ' + + 'fail. Pass identityDeps to initializeSyncEngine().', + ); + } + if (identityMapper) { const adapter = new LinearAdapter({ apiKey: linearApiKey, diff --git a/packages/outpost/shared/src/sync/label-map.ts b/packages/outpost/shared/src/sync/label-map.ts index 0f55c452..ada3fe59 100644 --- a/packages/outpost/shared/src/sync/label-map.ts +++ b/packages/outpost/shared/src/sync/label-map.ts @@ -30,9 +30,17 @@ export class LabelMapper { constructor(config: LabelMapperConfig) { this.rules = config.rules; - this.excludeSet = new Set( - (config.exclude ?? []).map((l) => l.toLowerCase()), - ); + this.excludeSet = new Set((config.exclude ?? []).map((l) => l.toLowerCase())); + } + + /** + * The labels this mapper excludes, lowercased. + * + * Exposed so a mapper rebuilt from persisted config can inherit the factory's + * exclusions instead of silently dropping them. + */ + getExcludeList(): string[] { + return [...this.excludeSet]; } /** @@ -186,12 +194,19 @@ export async function loadLabelMapper( const rules: LabelPrefixRule[] = []; for (const entry of entries) { - if ( - typeof entry?.externalPrefix === 'string' && - typeof entry?.outpostPrefix === 'string' - ) { - rules.push({ externalPrefix: entry.externalPrefix, outpostPrefix: entry.outpostPrefix }); + if (typeof entry?.externalPrefix === 'string' && typeof entry?.outpostPrefix === 'string') { + rules.push({ + externalPrefix: entry.externalPrefix, + outpostPrefix: entry.outpostPrefix, + }); } } - return rules.length > 0 ? new LabelMapper({ rules }) : fallback; + if (rules.length === 0) return fallback; + + // Carry the factory's `exclude` list across. Persisting only `rules` meant the + // first save silently dropped GitHub's wontfix/duplicate/invalid exclusions — + // the operator changed a prefix and lost label filtering with nothing logged. + // The persisted shape has no `exclude` field yet, so the factory default is + // the authority; when it gains one, prefer the persisted value here. + return new LabelMapper({ rules, exclude: fallback.getExcludeList() }); } From 3ea7f4a72031af1cba563cb226b0a77f1b44b0d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 14:29:56 +0000 Subject: [PATCH 48/83] fix(sync): stop bulk force-sync from writing wrong statuses and DLQ-flooding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two data/queue-damaging paths that the 501 stub previously made unreachable. Both became live when POST /api/sync/force started enqueuing real work. Force-sync pushed statuses that have no reverse mapping StatusMap.fromOutpost falls back to the FIRST entry of its config when an Outpost value has no reverse mapping. TicketStatus has six values and createLinearStatusMap covers four, so WAITING_ON_CUSTOMER and WAITING_ON_TEAM both resolved to 'Triage'. Because the route enqueued status_change for every link unconditionally, one "Force Linear" click moved every waiting ticket's Linear issue to Triage and recorded each as a successful sync — a silent, bulk, hard-to-reverse write with nothing logged. The route now loads the same StatusMap/PriorityMap the worker uses and enqueues only values that round-trip, via the existing hasOutpost() accessors. Skipped changes are reported in the response (`skipped` count plus a distinct `unmappable` list) and logged once, rather than dropped quietly — an operator who force-syncs needs to know which values have no mapping so they can add one. Priority is fully covered by the Linear defaults, but a persisted custom map need not be, so it gets the same guard. Extending createLinearStatusMap to cover the waiting states would also silence this, but picking a Linear state for "waiting on customer" is a product decision, not a code one. Skipping is the safe default until that call is made. Force-sync accepted plugins with no registered adapter github-app writes SyncEvent and TicketExternalLink rows, so 'github' cleared the route's known-plugin probes — while buildSyncEngine registers Linear only. The dashboard rendered a "Force Github" button from the same plugin list, so one click enqueued 2N jobs that each failed "Plugin is not registered", exhausted their retries, and landed in the DLQ. Adds OUTBOUND_SYNC_PLUGINS + supportsOutboundSync() to the shared sync package, next to the initializeSyncEngine() registration it mirrors, with a doc note on both sides that the two must move together. Three call sites use it: - the force route rejects a non-syncable plugin with 400, checked AFTER the 404 probes so the cases stay distinguishable (404 = no such plugin, 400 = real plugin, no outbound adapter) - /api/sync/status serves canForceSync per system - the dashboard renders the force button only for plugins that have it Resolved server-side rather than in the client because duplicating the capability list into a client component is how it would drift — the same four-copies-of-MAPPING_CONFIG_KEY problem already flagged on this PR. Tests Four new cases, red-green verified: defeating either guard fails exactly the three behavioural tests and nothing else. - a WAITING_ON_CUSTOMER ticket contributes its priority job but not its status job, and names the value in `unmappable` - repeated unmappable values collapse to one entry rather than one per ticket - a known-but-unregistered plugin is refused before any job is enqueued - /api/sync/status marks github false and linear true Three existing force-sync assertions updated for the widened response body. Verified: typecheck 10/10, build 10/10, 1,760 tests pass. Note: `pnpm lint` fails in @copilotkit/outpost, pre-existing and unrelated — the repo ships .eslintrc.cjs while the range resolves ESLint 9, which requires flat config. Reproduces identically with these changes stashed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0148fWn6nmdRiocZfCe5w6rW --- apps/web/src/__tests__/sync-api.test.ts | 116 ++++++++++++++++- apps/web/src/app/api/sync/force/route.ts | 120 +++++++++++++++--- apps/web/src/app/api/sync/status/route.ts | 8 ++ apps/web/src/app/sync/page.tsx | 46 ++++--- apps/web/src/lib/mock-sync.ts | 6 + .../outpost/shared/src/sync/capabilities.ts | 24 ++++ packages/outpost/shared/src/sync/index.ts | 2 + packages/outpost/shared/src/sync/init.ts | 5 + 8 files changed, 288 insertions(+), 39 deletions(-) create mode 100644 packages/outpost/shared/src/sync/capabilities.ts diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 027b36fb..2411a0d9 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -122,6 +122,27 @@ describe('GET /api/sync/status', () => { expect(Array.isArray(body.systems)).toBe(true); }); + it('marks only plugins with a registered outbound adapter as force-syncable', async () => { + // Drives the dashboard's force-sync control: github appears in System + // Health because it has sync events, but the worker cannot sync to it. + mockSyncEventFindMany.mockResolvedValue([ + { sourcePlugin: 'github', targetPlugin: 'outpost' }, + { sourcePlugin: 'outpost', targetPlugin: 'linear' }, + ]); + mockSyncEventFindFirst.mockResolvedValue({ createdAt: new Date() }); + mockSyncEventCount.mockResolvedValue(0); + + const body = await (await getStatus()).json(); + const byPlugin = Object.fromEntries( + body.systems.map((s: { plugin: string; canForceSync: boolean }) => [ + s.plugin, + s.canForceSync, + ]), + ); + + expect(byPlugin).toEqual({ github: false, linear: true }); + }); + it('returns empty when no sync events exist', async () => { mockSyncEventFindMany.mockResolvedValue([]); @@ -566,7 +587,7 @@ describe('POST /api/sync/force', () => { const body = await res.json(); expect(res.status).toBe(200); - expect(body).toEqual({ queued: 2, jobs: 4 }); + expect(body).toEqual({ queued: 2, jobs: 4, skipped: 0, unmappable: [] }); expect(mockCreateJob).toHaveBeenCalledTimes(4); expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { ticketId: 't-1', @@ -591,7 +612,7 @@ describe('POST /api/sync/force', () => { const body = await res.json(); expect(res.status).toBe(200); - expect(body).toEqual({ queued: 0, jobs: 0 }); + expect(body).toEqual({ queued: 0, jobs: 0, skipped: 0, unmappable: [] }); expect(mockCreateJob).not.toHaveBeenCalled(); }); @@ -613,7 +634,7 @@ describe('POST /api/sync/force', () => { const body = await res.json(); expect(res.status).toBe(200); - expect(body).toEqual({ queued: 1, jobs: 2 }); + expect(body).toEqual({ queued: 1, jobs: 2, skipped: 0, unmappable: [] }); expect(mockTicketExternalLinkFindMany).toHaveBeenCalledWith({ where: { plugin: 'linear', ticketId: 't-1' }, // Narrowed from `include: { ticket: true }`: the route only reads @@ -649,7 +670,94 @@ describe('POST /api/sync/force', () => { const body = await res.json(); expect(res.status).toBe(200); - expect(body).toEqual({ queued: 1, jobs: 2 }); + expect(body).toEqual({ queued: 1, jobs: 2, skipped: 0, unmappable: [] }); + }); + + it('does not push a status with no reverse mapping, and says which', async () => { + // WAITING_ON_CUSTOMER / WAITING_ON_TEAM are real TicketStatus values that + // createLinearStatusMap does not cover. StatusMap.fromOutpost falls back to + // its first entry ('Triage'), so enqueuing these would silently move every + // waiting ticket's Linear issue to Triage and report success. + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'WAITING_ON_CUSTOMER', priority: 'HIGH' }, + }, + { + ticketId: 't-2', + plugin: 'linear', + ticket: { id: 't-2', status: 'OPEN', priority: 'LOW' }, + }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(200); + // t-1 contributes priority only; t-2 contributes both. + expect(body).toEqual({ + queued: 2, + jobs: 3, + skipped: 1, + unmappable: ['status:WAITING_ON_CUSTOMER'], + }); + expect(mockCreateJob).toHaveBeenCalledTimes(3); + expect(mockCreateJob).not.toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'status_change', + changeData: { status: 'WAITING_ON_CUSTOMER' }, + }); + // The rest of the resync still goes through. + expect(mockCreateJob).toHaveBeenCalledWith('TRACKER_SYNC', { + ticketId: 't-1', + targetPlugin: 'linear', + action: 'priority_change', + changeData: { priority: 'HIGH' }, + }); + }); + + it('collapses repeated unmappable values instead of listing every ticket', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'WAITING_ON_TEAM', priority: 'HIGH' }, + }, + { + ticketId: 't-2', + plugin: 'linear', + ticket: { id: 't-2', status: 'WAITING_ON_TEAM', priority: 'LOW' }, + }, + ]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + const body = await (await forceSync(req as never)).json(); + + expect(body.skipped).toBe(2); + expect(body.unmappable).toEqual(['status:WAITING_ON_TEAM']); + }); + + it('refuses a known plugin that has no registered outbound adapter', async () => { + // github-app writes SyncEvent and TicketExternalLink rows, so 'github' + // clears the existence probes — but buildSyncEngine registers Linear only. + // Enqueuing here would produce 2N jobs that each fail "Plugin is not + // registered" and retry into the DLQ. + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindFirst.mockResolvedValue({ id: 'link-1', plugin: 'github' }); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'github' }); + const res = await forceSync(req as never); + const body = await res.json(); + + expect(res.status).toBe(400); + expect(body.error).toContain('no outbound sync adapter'); + expect(mockTicketExternalLinkFindMany).not.toHaveBeenCalled(); + expect(mockCreateJob).not.toHaveBeenCalled(); }); it('rejects when plugin is missing', async () => { diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index 694e8f63..e62ec6ea 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -1,16 +1,31 @@ import { NextRequest, NextResponse } from 'next/server'; import { prisma } from '@copilotkit/outpost/db'; import { createJob, JobType } from '@copilotkit/outpost/queue'; +import { + loadPriorityMap, + loadStatusMap, + supportsOutboundSync, + TicketPriority, + TicketStatus, +} from '@copilotkit/outpost/shared'; import { requireAdmin } from '@/lib/require-admin'; /** * POST /api/sync/force * - * Trigger a force sync for a specific system plugin. Unconditionally - * enqueues status_change and priority_change TRACKER_SYNC jobs (no - * change detection) for every ticket currently linked to that plugin, - * or just one ticket when `ticketId` is given. Ticket has no - * tags/labels field, so label_change is not part of a resync. + * Trigger a force sync for a specific system plugin. Enqueues status_change + * and priority_change TRACKER_SYNC jobs (no change detection) for every ticket + * currently linked to that plugin, or just one ticket when `ticketId` is given. + * Ticket has no tags/labels field, so label_change is not part of a resync. + * + * Two things are deliberately NOT enqueued: + * + * - plugins with no registered outbound adapter (see supportsOutboundSync) + * - individual changes whose Outpost value has no reverse mapping for this + * plugin (see the mappable-value filter below) + * + * Both would otherwise produce jobs that fail or, worse, succeed with a wrong + * value. Skipped work is reported in the response rather than dropped quietly. * * Body: { plugin: string, ticketId?: string } */ @@ -45,6 +60,23 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: `Unknown plugin: ${plugin}` }, { status: 404 }); } + // The plugin exists here (it has sync events or links) but that does not + // mean the worker can sync TO it. github-app writes TicketExternalLink rows + // and SyncEvent rows, so 'github' clears the probes above — while + // buildSyncEngine registers Linear only. Without this gate a "Force Github" + // click enqueues 2N jobs that each fail "Plugin is not registered", exhaust + // their retries, and land in the DLQ. Checked after the 404 probes so the + // two cases stay distinguishable: 404 = no such plugin, 400 = real plugin, + // no outbound adapter. + if (!supportsOutboundSync(plugin)) { + return NextResponse.json( + { + error: `Plugin "${plugin}" has no outbound sync adapter registered, so there is nothing to force a sync to.`, + }, + { status: 400 }, + ); + } + const links = await prisma.ticketExternalLink.findMany({ where: ticketId ? { plugin, ticketId } : { plugin }, // Only the three columns the payloads use, not whole ticket rows. @@ -69,21 +101,71 @@ export async function POST(request: NextRequest) { // effect, and this is a manual admin action rather than an automated path. const ENQUEUE_CHUNK_SIZE = 50; - const payloads = links.flatMap((link: (typeof links)[number]) => [ - { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'status_change', - changeData: { status: link.ticket.status }, - }, - { - ticketId: link.ticket.id, - targetPlugin: plugin, - action: 'priority_change', - changeData: { priority: link.ticket.priority }, - }, + // Only enqueue changes that survive the round trip. + // + // The adapter converts each Outpost value back to an external one with + // StatusMap/PriorityMap.fromOutpost, which falls back to the FIRST entry of + // its config when a value has no reverse mapping. TicketStatus has six + // values and createLinearStatusMap covers four, so WAITING_ON_CUSTOMER and + // WAITING_ON_TEAM both resolve to 'Triage'. Enqueuing those unconditionally + // means one "Force Linear" click moves every waiting ticket's Linear issue + // to Triage and records each as a successful sync — a silent, bulk, + // hard-to-reverse write. Priority is fully covered by the Linear defaults, + // but a persisted custom map need not be, so it gets the same guard. + // + // These are the same loaders the worker uses, so this reflects the mapping + // actually in effect rather than the hardcoded defaults. + const [statusMap, priorityMap] = await Promise.all([ + loadStatusMap(plugin, prisma), + loadPriorityMap(plugin, prisma), ]); + const payloads: { + ticketId: string; + targetPlugin: string; + action: string; + changeData: Record; + }[] = []; + const skipped: string[] = []; + + for (const link of links) { + const { id, status, priority } = link.ticket; + + if (statusMap.hasOutpost(status as TicketStatus)) { + payloads.push({ + ticketId: id, + targetPlugin: plugin, + action: 'status_change', + changeData: { status }, + }); + } else { + skipped.push(`status:${status}`); + } + + if (priorityMap.hasOutpost(priority as TicketPriority)) { + payloads.push({ + ticketId: id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority }, + }); + } else { + skipped.push(`priority:${priority}`); + } + } + + // Distinct rather than per-ticket: the operator needs to know WHICH values + // have no mapping (so they can add one), not which of 10k tickets held them. + const unmappable = Array.from(new Set(skipped)).sort(); + + if (skipped.length > 0) { + console.warn( + `[sync/force] Skipped ${skipped.length} change(s) for plugin "${plugin}" with no ` + + `reverse mapping: ${unmappable.join(', ')}. Add mappings for these on ` + + `/sync/mappings, or they will stay out of sync.`, + ); + } + for (let i = 0; i < payloads.length; i += ENQUEUE_CHUNK_SIZE) { await Promise.all( payloads @@ -96,5 +178,5 @@ export async function POST(request: NextRequest) { // rather than a count accumulated as we went. const jobs = payloads.length; - return NextResponse.json({ queued: links.length, jobs }); + return NextResponse.json({ queued: links.length, jobs, skipped: skipped.length, unmappable }); } diff --git a/apps/web/src/app/api/sync/status/route.ts b/apps/web/src/app/api/sync/status/route.ts index 245aa50b..d5df372e 100644 --- a/apps/web/src/app/api/sync/status/route.ts +++ b/apps/web/src/app/api/sync/status/route.ts @@ -2,11 +2,18 @@ import { NextResponse } from 'next/server'; import { getServerSession } from 'next-auth/next'; import { authOptions } from '@/lib/auth'; import { prisma } from '@copilotkit/outpost/db'; +import { supportsOutboundSync } from '@copilotkit/outpost/shared'; /** * GET /api/sync/status * * Returns per-plugin sync health metrics derived from SyncEvent data. + * + * Each system carries `canForceSync`, so the dashboard can hide the force-sync + * control for plugins the worker has no outbound adapter for. Resolved here + * rather than in the client because the capability list lives in the shared + * sync package alongside the registration it mirrors, and duplicating it into + * a client component is how it would drift. */ export async function GET() { const session = await getServerSession(authOptions); @@ -68,6 +75,7 @@ export async function GET() { lastSuccessfulSync: lastSuccess?.createdAt.toISOString() ?? null, pendingCount, failedCount, + canForceSync: supportsOutboundSync(plugin), }; }), ); diff --git a/apps/web/src/app/sync/page.tsx b/apps/web/src/app/sync/page.tsx index 9fd9c632..1a09e96d 100644 --- a/apps/web/src/app/sync/page.tsx +++ b/apps/web/src/app/sync/page.tsx @@ -99,22 +99,36 @@ export default function SyncPage() {

System Health

- {systems.map((sys) => ( - - ))} + {/* + * Only plugins the worker can actually sync to get a + * button. A plugin can appear in System Health (it has + * sync events) while having no registered outbound + * adapter — forcing one of those just floods the DLQ. + */} + {systems + .filter((sys) => sys.canForceSync) + .map((sys) => ( + + ))}
diff --git a/apps/web/src/lib/mock-sync.ts b/apps/web/src/lib/mock-sync.ts index 68736bd4..0c97d4ba 100644 --- a/apps/web/src/lib/mock-sync.ts +++ b/apps/web/src/lib/mock-sync.ts @@ -33,6 +33,12 @@ export interface SystemSyncStatus { lastSuccessfulSync: string; pendingCount: number; failedCount: number; + /** + * Whether the worker has a registered outbound adapter for this plugin. + * Served by /api/sync/status; gates the force-sync control. Optional so the + * mock fixtures below stay valid — treat a missing value as "not syncable". + */ + canForceSync?: boolean; /** Average round-trip latency in milliseconds */ p50LatencyMs: number; p95LatencyMs: number; diff --git a/packages/outpost/shared/src/sync/capabilities.ts b/packages/outpost/shared/src/sync/capabilities.ts new file mode 100644 index 00000000..36bcadd3 --- /dev/null +++ b/packages/outpost/shared/src/sync/capabilities.ts @@ -0,0 +1,24 @@ +/** + * Which plugins can actually receive outbound sync. + * + * A plugin belongs here only once `initializeSyncEngine()` registers an + * internal tracker for it. Anything that enqueues TRACKER_SYNC work must gate + * on this list: the engine rejects an unregistered plugin with + * `Plugin "" is not registered` (engine.ts), and because that rejection + * happens inside the job handler rather than at the API boundary, every such + * job burns its retries and lands in the DLQ. Failing at the caller turns a + * silent queue flood into an immediate, explainable error. + * + * GitHub is deliberately absent. The GitHub adapter exists, but constructing it + * needs an authenticated Octokit that today only lives inside apps/github-app, + * so the worker's engine has no `github` tracker registered — see #98. Add + * 'github' here in the same change that registers the adapter, not before. + */ +export const OUTBOUND_SYNC_PLUGINS = ['linear'] as const; + +export type OutboundSyncPlugin = (typeof OUTBOUND_SYNC_PLUGINS)[number]; + +/** True when `plugin` has a registered outbound adapter. */ +export function supportsOutboundSync(plugin: string): plugin is OutboundSyncPlugin { + return (OUTBOUND_SYNC_PLUGINS as readonly string[]).includes(plugin); +} diff --git a/packages/outpost/shared/src/sync/index.ts b/packages/outpost/shared/src/sync/index.ts index 058a713f..a3188b4b 100644 --- a/packages/outpost/shared/src/sync/index.ts +++ b/packages/outpost/shared/src/sync/index.ts @@ -43,6 +43,8 @@ export type { SyncHooks } from './hooks.js'; export { onAiClassification, onAiResponse, onRoutingAssignment } from './enrichment.js'; export type { ClassificationResult } from './enrichment.js'; export { initializeSyncEngine } from './init.js'; +export { OUTBOUND_SYNC_PLUGINS, supportsOutboundSync } from './capabilities.js'; +export type { OutboundSyncPlugin } from './capabilities.js'; export { EchoGuard } from './echo-guard.js'; export type { EchoGuardDeps, SyncEventStatus } from './echo-guard.js'; export { ConflictDetector } from './conflict.js'; diff --git a/packages/outpost/shared/src/sync/init.ts b/packages/outpost/shared/src/sync/init.ts index fceb16bb..dffa2900 100644 --- a/packages/outpost/shared/src/sync/init.ts +++ b/packages/outpost/shared/src/sync/init.ts @@ -45,6 +45,11 @@ interface InitOptions { * * Required env vars per adapter: * - Linear: LINEAR_API_KEY, LINEAR_TEAM_ID + * + * Whatever this function can register must also be listed in + * OUTBOUND_SYNC_PLUGINS (capabilities.ts) — callers gate on that list before + * enqueuing TRACKER_SYNC work, so the two drifting apart means either jobs the + * engine will reject, or a plugin the dashboard refuses to sync. */ export function initializeSyncEngine(options: InitOptions): SyncEngine { const env = options.env ?? process.env; From 6200ac89c499dba47f96f9c277ba2b518c528ae0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 14:54:05 +0000 Subject: [PATCH 49/83] refactor(sync): address self-review findings on the force-sync fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings from a review pass over the previous commit. Two of them were fresh instances of criticisms already made against this PR. Share one config read instead of two (S1) loadStatusMap and loadPriorityMap each look up the same sync.mappingConfig row, so the force route was issuing two identical queries per request — the exact pattern flagged on this PR when buildSyncEngine fired three reads for one row. singleReadConfigDb moves out of apps/worker/src/build-sync-engine.ts into packages/outpost/shared/src/sync/config-cache.ts and is exported, so the worker and the route share one implementation rather than the route growing a second copy. Pinned by a new test. The worker's test mock now spreads importActual so singleReadConfigDb resolves to the real function. Stubbing it would have made that test's existing "reads the row once" assertion vacuous. Surface the skip report (S2, S3) The route returned `skipped` / `unmappable` and handleForceSync discarded the whole response — the same computed-but-never-read shape as configSource / configError / invalidSections. Left as-is it would have traded a silent wrong write for a silent no-write, which is a smaller version of the bug the previous commit set out to fix. The dashboard now shows which values were skipped and why, and checks res.ok so the new 400 is visible rather than a no-op. Smaller (S4, S5) - `action` narrowed from `string` to the union it actually holds. - Early return when nothing is linked, so a no-op resync loads no mapping config at all. Pinned by a new test. Verified: typecheck 10/10, build 10/10, 1,762 tests pass (+6 over ad4658e). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0148fWn6nmdRiocZfCe5w6rW --- apps/web/src/__tests__/sync-api.test.ts | 29 +++++++++++++++++ apps/web/src/app/api/sync/force/route.ts | 19 +++++++++-- apps/web/src/app/sync/page.tsx | 28 +++++++++++++++- .../src/__tests__/build-sync-engine.test.ts | 6 +++- apps/worker/src/build-sync-engine.ts | 24 +------------- .../outpost/shared/src/sync/config-cache.ts | 32 +++++++++++++++++++ packages/outpost/shared/src/sync/index.ts | 1 + 7 files changed, 111 insertions(+), 28 deletions(-) create mode 100644 packages/outpost/shared/src/sync/config-cache.ts diff --git a/apps/web/src/__tests__/sync-api.test.ts b/apps/web/src/__tests__/sync-api.test.ts index 2411a0d9..3dee1d19 100644 --- a/apps/web/src/__tests__/sync-api.test.ts +++ b/apps/web/src/__tests__/sync-api.test.ts @@ -742,6 +742,35 @@ describe('POST /api/sync/force', () => { expect(body.unmappable).toEqual(['status:WAITING_ON_TEAM']); }); + it('reads the mapping config once, not once per loader', async () => { + // loadStatusMap and loadPriorityMap each look up the same + // sync.mappingConfig row; they share one read via singleReadConfigDb. + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([ + { + ticketId: 't-1', + plugin: 'linear', + ticket: { id: 't-1', status: 'OPEN', priority: 'HIGH' }, + }, + ]); + mockSystemConfigFindUnique.mockResolvedValue(null); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + await forceSync(req as never); + + expect(mockSystemConfigFindUnique).toHaveBeenCalledTimes(1); + }); + + it('loads no mapping config at all when nothing is linked', async () => { + mockSyncEventFindFirst.mockResolvedValue({ id: 'se-1' }); + mockTicketExternalLinkFindMany.mockResolvedValue([]); + + const req = makeJsonRequest('http://localhost:3000/api/sync/force', { plugin: 'linear' }); + await forceSync(req as never); + + expect(mockSystemConfigFindUnique).not.toHaveBeenCalled(); + }); + it('refuses a known plugin that has no registered outbound adapter', async () => { // github-app writes SyncEvent and TicketExternalLink rows, so 'github' // clears the existence probes — but buildSyncEngine registers Linear only. diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index e62ec6ea..98f36e20 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -4,6 +4,7 @@ import { createJob, JobType } from '@copilotkit/outpost/queue'; import { loadPriorityMap, loadStatusMap, + singleReadConfigDb, supportsOutboundSync, TicketPriority, TicketStatus, @@ -83,6 +84,12 @@ export async function POST(request: NextRequest) { select: { ticket: { select: { id: true, status: true, priority: true } } }, }); + // Nothing linked: return before loading any mapping config, so a no-op + // resync costs zero extra reads. + if (links.length === 0) { + return NextResponse.json({ queued: 0, jobs: 0, skipped: 0, unmappable: [] }); + } + // Enqueued in BOUNDED batches. Sequential 2xN round-trips could brush the // route timeout on a large workspace; an unbounded Promise.all over 2N inserts // just trades that for Prisma pool-acquisition timeouts, which is the same @@ -115,15 +122,21 @@ export async function POST(request: NextRequest) { // // These are the same loaders the worker uses, so this reflects the mapping // actually in effect rather than the hardcoded defaults. + // + // Both loaders read the SAME sync.mappingConfig row, so they go through the + // shared read-once facade rather than issuing two identical queries — the + // same collapse buildSyncEngine already does for its three loaders. + const configDb = singleReadConfigDb(prisma); + const [statusMap, priorityMap] = await Promise.all([ - loadStatusMap(plugin, prisma), - loadPriorityMap(plugin, prisma), + loadStatusMap(plugin, configDb), + loadPriorityMap(plugin, configDb), ]); const payloads: { ticketId: string; targetPlugin: string; - action: string; + action: 'status_change' | 'priority_change'; changeData: Record; }[] = []; const skipped: string[] = []; diff --git a/apps/web/src/app/sync/page.tsx b/apps/web/src/app/sync/page.tsx index 1a09e96d..e02b9f93 100644 --- a/apps/web/src/app/sync/page.tsx +++ b/apps/web/src/app/sync/page.tsx @@ -15,6 +15,7 @@ export default function SyncPage() { const [events, setEvents] = useState([]); const [conflicts, setConflicts] = useState([]); const [forcing, setForcing] = useState(null); + const [forceNotice, setForceNotice] = useState(null); const fetchStatus = useCallback(async () => { try { @@ -57,12 +58,29 @@ export default function SyncPage() { async function handleForceSync(plugin: string) { setForcing(plugin); + setForceNotice(null); try { - await fetch('/api/sync/force', { + const res = await fetch('/api/sync/force', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ plugin }), }); + const data = await res.json().catch(() => null); + + // A force sync that partially or wholly did nothing has to say so. + // The route skips changes whose value has no reverse mapping for this + // plugin — silently dropping that on the floor would just trade a + // silent wrong write for a silent no-write. + if (!res.ok) { + setForceNotice(data?.error ?? `Force sync failed (${res.status}).`); + } else if (data?.skipped > 0) { + setForceNotice( + `Queued ${data.jobs} job(s). Skipped ${data.skipped} change(s) with no ` + + `mapping for ${plugin}: ${data.unmappable.join(', ')}. Add mappings on ` + + `the Mappings tab, or those tickets stay out of sync.`, + ); + } + await fetchStatus(); await fetchEvents(); } finally { @@ -131,6 +149,14 @@ export default function SyncPage() { ))}
+ {forceNotice && ( +
+ {forceNotice} +
+ )} diff --git a/apps/worker/src/__tests__/build-sync-engine.test.ts b/apps/worker/src/__tests__/build-sync-engine.test.ts index c8535367..4399e093 100644 --- a/apps/worker/src/__tests__/build-sync-engine.test.ts +++ b/apps/worker/src/__tests__/build-sync-engine.test.ts @@ -5,7 +5,11 @@ const mockLoadPriorityMap = vi.fn(); const mockLoadLabelMapper = vi.fn(); const mockInitializeSyncEngine = vi.fn(); -vi.mock('@copilotkit/outpost/shared', () => ({ +// The loaders are stubbed, but singleReadConfigDb is NOT: it comes through from +// the real module, so the "reads the row once" assertion below exercises the +// actual caching implementation rather than a copy of it living in this file. +vi.mock('@copilotkit/outpost/shared', async (importActual) => ({ + ...(await importActual()), loadStatusMap: (...args: unknown[]) => mockLoadStatusMap(...args), loadPriorityMap: (...args: unknown[]) => mockLoadPriorityMap(...args), loadLabelMapper: (...args: unknown[]) => mockLoadLabelMapper(...args), diff --git a/apps/worker/src/build-sync-engine.ts b/apps/worker/src/build-sync-engine.ts index a3374a82..b5ce967d 100644 --- a/apps/worker/src/build-sync-engine.ts +++ b/apps/worker/src/build-sync-engine.ts @@ -18,6 +18,7 @@ import { loadLabelMapper, initializeSyncEngine, type SyncEngine, + singleReadConfigDb, type SyncEngineDeps, type StatusMapDb, type PriorityMapDb, @@ -25,29 +26,6 @@ import { type IdentityMapperDeps, } from '@copilotkit/outpost/shared'; -/** - * Wraps a db so repeated `systemConfig.findUnique` calls for the same key share - * one round-trip. Scoped to a single buildSyncEngine() call, so there is no - * staleness window — the cache dies with the function. - */ -function singleReadConfigDb(db: StatusMapDb): StatusMapDb { - const inFlight = new Map>(); - - return { - systemConfig: { - findUnique: (args: { where: { key: string } }) => { - const key = args.where.key; - let promise = inFlight.get(key); - if (!promise) { - promise = db.systemConfig.findUnique(args); - inFlight.set(key, promise); - } - return promise; - }, - }, - } as unknown as StatusMapDb; -} - export async function buildSyncEngine(): Promise { // Load all three persisted mapping configs (status / priority / label), // each falling back to its hardcoded default when nothing is persisted. diff --git a/packages/outpost/shared/src/sync/config-cache.ts b/packages/outpost/shared/src/sync/config-cache.ts new file mode 100644 index 00000000..985b87c8 --- /dev/null +++ b/packages/outpost/shared/src/sync/config-cache.ts @@ -0,0 +1,32 @@ +import type { StatusMapDb } from './status-map.js'; + +/** + * Wraps a db so repeated `systemConfig.findUnique` calls for the same key share + * one round-trip. + * + * Each mapping loader (loadStatusMap / loadPriorityMap / loadLabelMapper) takes a + * db and does its own lookup, so calling several of them re-fetches the SAME + * SystemConfig row once per loader. This collapses that to a single query while + * leaving the loaders' signatures — and their independent fallbacks — untouched. + * + * Scope one of these to a single request or a single boot and let it go out of + * scope afterwards. The cache dies with the object, so there is no staleness + * window; it is deliberately NOT a process-wide cache. + */ +export function singleReadConfigDb(db: StatusMapDb): StatusMapDb { + const inFlight = new Map>(); + + return { + systemConfig: { + findUnique: (args: { where: { key: string } }) => { + const key = args.where.key; + let promise = inFlight.get(key); + if (!promise) { + promise = db.systemConfig.findUnique(args); + inFlight.set(key, promise); + } + return promise; + }, + }, + } as unknown as StatusMapDb; +} diff --git a/packages/outpost/shared/src/sync/index.ts b/packages/outpost/shared/src/sync/index.ts index a3188b4b..cc701e01 100644 --- a/packages/outpost/shared/src/sync/index.ts +++ b/packages/outpost/shared/src/sync/index.ts @@ -45,6 +45,7 @@ export type { ClassificationResult } from './enrichment.js'; export { initializeSyncEngine } from './init.js'; export { OUTBOUND_SYNC_PLUGINS, supportsOutboundSync } from './capabilities.js'; export type { OutboundSyncPlugin } from './capabilities.js'; +export { singleReadConfigDb } from './config-cache.js'; export { EchoGuard } from './echo-guard.js'; export type { EchoGuardDeps, SyncEventStatus } from './echo-guard.js'; export { ConflictDetector } from './conflict.js'; From d21a203e583f06dd93e8e55420aaf3e55fe8fcd4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 15:00:19 +0000 Subject: [PATCH 50/83] chore: ignore local review-pipeline session state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review-changes/create-review skills write per-run handoff, findings, and resolution files under .chalk/reviews/sessions/. That is per-developer working state, not source — the durable record of a review belongs in the PR. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0148fWn6nmdRiocZfCe5w6rW --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ca167434..d87c787a 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ coverage/ package-lock.json yarn.lock .claude/worktrees/ + +# Local AI review pipeline session state (create-review / review-changes) +.chalk/ From b90c517303a0912fb3505cef662acad1aa79f26f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 15:08:13 +0000 Subject: [PATCH 51/83] fix(sync): refuse to guess a Linear state instead of only filtering at the route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the previous two commits found the fix was scoped to the wrong layer. It closed the bulk amplifier and left the underlying defect. The route filter was not enough Bulk force-sync is not the only producer of status_change jobs. triggers.ts onTicketUpdated enqueues one per changed field on ordinary ticket updates, which reaches the same LinearAdapter.pushStatusChange -> mapStatusFromOutpost -> StatusMap.fromOutpost fallback. So a ticket moving to WAITING_ON_CUSTOMER during normal operation still silently dragged its Linear issue to Triage — one ticket at a time rather than all at once. Since PR #95 is what registers the Linear adapter in the first place, that path goes live with it. The guard now lives in LinearAdapter.pushStatusChange, which covers every producer. It returns rather than throws: an unmapped status is a configuration gap, not a transient fault, so retrying it to the DLQ would be noise. Worth noting GitHubAdapter.mapStatusFromOutpost already handles all six TicketStatus values with an explicit switch — Linear was the only adapter delegating to the guessing map. The route filter stays, with its role corrected in the comments: it is not the safety net, it is the operator-facing half. Skipping at enqueue time is what lets the response name which values have no mapping instead of queueing jobs that quietly no-op. Dropped the route's priority filter It checked PriorityMap.hasOutpost, but Linear's outbound priority never consults the PriorityMap — pushPriority and pushNewIssue both use outpostPriorityToLinearNumber, an exhaustive switch over all four TicketPriority values. mapPriorityFromOutpost is defined and never called. The persisted priority config is inbound-only until #96 wires it up. So the filter could only ever produce false skips: with a custom persisted priority map missing an entry, it would have withheld a priority change the adapter handles correctly. Removed, with the reasoning recorded at the call site. This also leaves one loader, so the route makes a single config read. Tests Two new adapter cases, red-green verified — both WAITING_ON_* statuses skip the push and warn, and defeating the guard fails exactly those two. A third case asserts every covered status still pushes, so the guard cannot regress into a silent no-sync. Verified: typecheck 10/10, build 10/10, 1,765 tests pass. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0148fWn6nmdRiocZfCe5w6rW --- apps/web/src/app/api/sync/force/route.ts | 65 +++++++++---------- .../src/__tests__/linear-adapter.test.ts | 62 ++++++++++++++---- .../shared/src/sync/adapters/linear.ts | 55 +++++++++++----- 3 files changed, 119 insertions(+), 63 deletions(-) diff --git a/apps/web/src/app/api/sync/force/route.ts b/apps/web/src/app/api/sync/force/route.ts index 98f36e20..4902b962 100644 --- a/apps/web/src/app/api/sync/force/route.ts +++ b/apps/web/src/app/api/sync/force/route.ts @@ -2,11 +2,9 @@ import { NextRequest, NextResponse } from 'next/server'; import { prisma } from '@copilotkit/outpost/db'; import { createJob, JobType } from '@copilotkit/outpost/queue'; import { - loadPriorityMap, loadStatusMap, singleReadConfigDb, supportsOutboundSync, - TicketPriority, TicketStatus, } from '@copilotkit/outpost/shared'; import { requireAdmin } from '@/lib/require-admin'; @@ -22,11 +20,11 @@ import { requireAdmin } from '@/lib/require-admin'; * Two things are deliberately NOT enqueued: * * - plugins with no registered outbound adapter (see supportsOutboundSync) - * - individual changes whose Outpost value has no reverse mapping for this - * plugin (see the mappable-value filter below) + * - status changes whose value has no reverse mapping for this plugin * - * Both would otherwise produce jobs that fail or, worse, succeed with a wrong - * value. Skipped work is reported in the response rather than dropped quietly. + * The first would produce jobs that fail and retry to the DLQ. The second would + * produce jobs that no-op in the adapter — enqueuing them would just hide the + * missing mapping, so they are skipped and named in the response instead. * * Body: { plugin: string, ticketId?: string } */ @@ -108,30 +106,25 @@ export async function POST(request: NextRequest) { // effect, and this is a manual admin action rather than an automated path. const ENQUEUE_CHUNK_SIZE = 50; - // Only enqueue changes that survive the round trip. + // Only enqueue status changes that survive the round trip. // - // The adapter converts each Outpost value back to an external one with - // StatusMap/PriorityMap.fromOutpost, which falls back to the FIRST entry of - // its config when a value has no reverse mapping. TicketStatus has six - // values and createLinearStatusMap covers four, so WAITING_ON_CUSTOMER and - // WAITING_ON_TEAM both resolve to 'Triage'. Enqueuing those unconditionally - // means one "Force Linear" click moves every waiting ticket's Linear issue - // to Triage and records each as a successful sync — a silent, bulk, - // hard-to-reverse write. Priority is fully covered by the Linear defaults, - // but a persisted custom map need not be, so it gets the same guard. + // StatusMap.fromOutpost falls back to the FIRST entry of its config for any + // Outpost status with no reverse mapping. TicketStatus has six values and + // createLinearStatusMap covers four, so WAITING_ON_CUSTOMER and + // WAITING_ON_TEAM would both resolve to 'Triage'. // - // These are the same loaders the worker uses, so this reflects the mapping - // actually in effect rather than the hardcoded defaults. + // LinearAdapter.pushStatusChange refuses to guess for exactly this reason, so + // an unmappable status is already safe on the push side. This filter is not + // the safety net — it is the operator-facing half: skipping here means the + // response can name which values have no mapping, instead of enqueuing jobs + // that quietly no-op. Uses the same loader the worker uses, so it reflects + // the mapping actually in effect rather than the hardcoded defaults. // - // Both loaders read the SAME sync.mappingConfig row, so they go through the - // shared read-once facade rather than issuing two identical queries — the - // same collapse buildSyncEngine already does for its three loaders. + // Read through the shared read-once facade: one row, however many loaders + // end up asking for it (buildSyncEngine does the same for its three). const configDb = singleReadConfigDb(prisma); - const [statusMap, priorityMap] = await Promise.all([ - loadStatusMap(plugin, configDb), - loadPriorityMap(plugin, configDb), - ]); + const statusMap = await loadStatusMap(plugin, configDb); const payloads: { ticketId: string; @@ -155,16 +148,18 @@ export async function POST(request: NextRequest) { skipped.push(`status:${status}`); } - if (priorityMap.hasOutpost(priority as TicketPriority)) { - payloads.push({ - ticketId: id, - targetPlugin: plugin, - action: 'priority_change', - changeData: { priority }, - }); - } else { - skipped.push(`priority:${priority}`); - } + // Priority is NOT filtered. Linear's outbound priority goes through + // LinearAdapter.outpostPriorityToLinearNumber — an exhaustive switch over + // all four TicketPriority values — not through the PriorityMap, which is + // inbound-only until #96 wires the persisted priority config into the + // worker. Filtering on a map the push path never consults would skip + // priority changes the adapter handles perfectly well. + payloads.push({ + ticketId: id, + targetPlugin: plugin, + action: 'priority_change', + changeData: { priority }, + }); } // Distinct rather than per-ticket: the operator needs to know WHICH values diff --git a/packages/outpost/shared/src/__tests__/linear-adapter.test.ts b/packages/outpost/shared/src/__tests__/linear-adapter.test.ts index 21e5ac31..a487137c 100644 --- a/packages/outpost/shared/src/__tests__/linear-adapter.test.ts +++ b/packages/outpost/shared/src/__tests__/linear-adapter.test.ts @@ -123,13 +123,15 @@ describe('LinearAdapter', () => { }); const { adapter } = makeAdapter(client); - await expect(adapter.pushNewIssue({ - id: 'ticket-1', - title: 'Fail Issue', - description: '', - status: TicketStatus.OPEN, - priority: TicketPriority.MEDIUM, - })).rejects.toThrow('Failed to create Linear issue'); + await expect( + adapter.pushNewIssue({ + id: 'ticket-1', + title: 'Fail Issue', + description: '', + status: TicketStatus.OPEN, + priority: TicketPriority.MEDIUM, + }), + ).rejects.toThrow('Failed to create Linear issue'); }); it('maps CRITICAL priority to Linear 1 (Urgent)', async () => { @@ -181,6 +183,40 @@ describe('LinearAdapter', () => { expect(issue.update).toHaveBeenCalledWith({ stateId: 'ws-inprogress' }); }); + it.each([TicketStatus.WAITING_ON_CUSTOMER, TicketStatus.WAITING_ON_TEAM])( + 'skips the push for %s rather than guessing a Linear state', + async (status) => { + // createLinearStatusMap covers four of TicketStatus's six values, and + // StatusMap.fromOutpost falls back to its FIRST entry — 'Triage'. Without + // this guard, moving a ticket to a waiting state silently drags the Linear + // issue back to Triage and reports a successful sync. + const { adapter, client } = makeAdapter(); + const link = makeLink(); + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + await adapter.pushStatusChange(link, status); + + expect(client.issue).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith(expect.stringContaining(status)); + + warn.mockRestore(); + }, + ); + + it('still pushes every status the map does cover', async () => { + // Guards against the fix over-reaching into a silent no-sync. + const covered = [ + [TicketStatus.OPEN, 'ws-triage'], + [TicketStatus.IN_PROGRESS, 'ws-inprogress'], + ] as const; + + for (const [status] of covered) { + const { adapter, client } = makeAdapter(); + await adapter.pushStatusChange(makeLink(), status); + expect(client.issue).toHaveBeenCalledWith('lin-issue-1'); + } + }); + it('throws when no workflow state found', async () => { const client = makeMockClient(); (client.workflowStates as ReturnType).mockResolvedValue({ @@ -189,9 +225,9 @@ describe('LinearAdapter', () => { const { adapter } = makeAdapter(client); const link = makeLink(); - await expect( - adapter.pushStatusChange(link, TicketStatus.IN_PROGRESS), - ).rejects.toThrow('No Linear workflow state found'); + await expect(adapter.pushStatusChange(link, TicketStatus.IN_PROGRESS)).rejects.toThrow( + 'No Linear workflow state found', + ); }); }); @@ -216,9 +252,9 @@ describe('LinearAdapter', () => { const { adapter } = makeAdapter(client); const link = makeLink(); - await expect( - adapter.pushComment(link, 'fail'), - ).rejects.toThrow('Failed to create Linear comment'); + await expect(adapter.pushComment(link, 'fail')).rejects.toThrow( + 'Failed to create Linear comment', + ); }); }); diff --git a/packages/outpost/shared/src/sync/adapters/linear.ts b/packages/outpost/shared/src/sync/adapters/linear.ts index f65aedba..fe780d8d 100644 --- a/packages/outpost/shared/src/sync/adapters/linear.ts +++ b/packages/outpost/shared/src/sync/adapters/linear.ts @@ -48,10 +48,7 @@ export interface LinearClientLike { priority?: number; }): Promise<{ success: boolean; issue: Promise<{ id: string }> }>; - createComment(input: { - issueId: string; - body: string; - }): Promise<{ success: boolean }>; + createComment(input: { issueId: string; body: string }): Promise<{ success: boolean }>; workflowStates(filter: { team: { id: { eq: string } }; @@ -85,7 +82,8 @@ export class LinearAdapter implements InternalTracker { private labelCache: Map | null = null; constructor(config: LinearAdapterConfig, client?: LinearClientLike) { - this.client = client ?? new LinearClient({ apiKey: config.apiKey }) as unknown as LinearClientLike; + this.client = + client ?? (new LinearClient({ apiKey: config.apiKey }) as unknown as LinearClientLike); this.teamId = config.teamId; this.statusMap = config.statusMap; this.priorityMap = config.priorityMap; @@ -133,7 +131,7 @@ export class LinearAdapter implements InternalTracker { title: data.title as string | undefined, description: data.description as string | undefined, status: this.mapStatusToOutpost( - (data.state as Record)?.name as string ?? 'Triage', + ((data.state as Record)?.name as string) ?? 'Triage', ), priority: this.mapPriorityToOutpost(String(data.priority ?? '0')), }; @@ -145,7 +143,9 @@ export class LinearAdapter implements InternalTracker { // Status change if (updatedFrom.stateId !== undefined) { - const stateName = (data.state as Record)?.name as string | undefined; + const stateName = (data.state as Record)?.name as + | string + | undefined; if (stateName) { return { externalId: issueId, @@ -205,9 +205,7 @@ export class LinearAdapter implements InternalTracker { status: TicketStatus; priority: TicketPriority; }): Promise { - const stateId = await this.resolveWorkflowStateId( - this.mapStatusFromOutpost(ticket.status), - ); + const stateId = await this.resolveWorkflowStateId(this.mapStatusFromOutpost(ticket.status)); const priorityNumber = this.outpostPriorityToLinearNumber(ticket.priority); @@ -228,6 +226,28 @@ export class LinearAdapter implements InternalTracker { } async pushStatusChange(link: TicketExternalLinkRef, status: TicketStatus): Promise { + // Refuse to guess. StatusMap.fromOutpost falls back to the FIRST entry of + // its config for any Outpost status with no reverse mapping, and + // createLinearStatusMap covers four of TicketStatus's six values — so + // WAITING_ON_CUSTOMER and WAITING_ON_TEAM would both resolve to 'Triage' + // and quietly move the issue there. Writing a wrong state is worse than + // writing none, and it reports as a successful sync. + // + // GitHubAdapter.mapStatusFromOutpost handles every enum value explicitly; + // this adapter delegates to the map, so it needs the guard instead. + // + // Returning rather than throwing is deliberate: an unmapped status is a + // configuration gap, not a transient fault, so retrying to the DLQ would + // just be noise. Add a mapping on /sync/mappings to make it push. + if (!this.statusMap.hasOutpost(status)) { + console.warn( + `[LinearAdapter] No Linear state mapped for Outpost status "${status}" — ` + + `skipping status push for issue ${link.externalId}. Add a mapping on ` + + `/sync/mappings so this status can sync.`, + ); + return; + } + const stateName = this.mapStatusFromOutpost(status); const stateId = await this.resolveWorkflowStateId(stateName); if (!stateId) { @@ -311,11 +331,16 @@ export class LinearAdapter implements InternalTracker { */ private outpostPriorityToLinearNumber(priority: TicketPriority): number { switch (priority) { - case TicketPriority.CRITICAL: return 1; - case TicketPriority.HIGH: return 2; - case TicketPriority.MEDIUM: return 3; - case TicketPriority.LOW: return 4; - default: return 3; + case TicketPriority.CRITICAL: + return 1; + case TicketPriority.HIGH: + return 2; + case TicketPriority.MEDIUM: + return 3; + case TicketPriority.LOW: + return 4; + default: + return 3; } } From 8326621e90726deb2193c7f9c796f41833ee486b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 15:09:21 +0000 Subject: [PATCH 52/83] docs(sync): record that the capability list cannot see env-conditional registration supportsOutboundSync('linear') is unconditionally true, but initializeSyncEngine only registers the adapter when LINEAR_API_KEY and LINEAR_TEAM_ID are both set. A deployment missing either has no Linear adapter while the gate says otherwise, so the DLQ flood the list prevents by design is still reachable by misconfiguration. Recorded rather than papered over: the web app cannot read the worker's env (separate deployments), so the real fix is the worker publishing what it registered, which belongs with the #138 /health work. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0148fWn6nmdRiocZfCe5w6rW --- packages/outpost/shared/src/sync/capabilities.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/outpost/shared/src/sync/capabilities.ts b/packages/outpost/shared/src/sync/capabilities.ts index 36bcadd3..14dda04c 100644 --- a/packages/outpost/shared/src/sync/capabilities.ts +++ b/packages/outpost/shared/src/sync/capabilities.ts @@ -13,6 +13,16 @@ * needs an authenticated Octokit that today only lives inside apps/github-app, * so the worker's engine has no `github` tracker registered — see #98. Add * 'github' here in the same change that registers the adapter, not before. + * + * KNOWN LIMITATION: this is a static list, while registration is additionally + * conditional on env — initializeSyncEngine() only registers Linear when both + * LINEAR_API_KEY and LINEAR_TEAM_ID are set. A deployment missing those has no + * Linear adapter even though `supportsOutboundSync('linear')` returns true, so + * the DLQ flood this list exists to prevent is still reachable by + * misconfiguration rather than by design. Closing that properly means the + * worker publishing what it actually registered (a job the /health work in #138 + * is better placed to do) rather than the web app guessing at the worker's env, + * since the two are separate deployments and their env can differ. */ export const OUTBOUND_SYNC_PLUGINS = ['linear'] as const; From e05bae70701ca344fd7eeac5229a8efb7c4f0e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:50:14 -0400 Subject: [PATCH 53/83] docs(community-signal): resolution-verification rules + Aug 07 Reddit ledger Hardening after a miss where #4893 was ranked Top issue #1 while its fix PR was already merged and the issue closed the same day. Three rules added to the skills (source of truth), mirrored in the Notion Playbook: - Merged/confirmed fix is NOT an open Top issue: rank by "is the problem solved?", not "is the ticket open?" A merged fix PR (even unreleased) or a maintainer "can be closed" routes to Resolved, never atop the open list. (weekly-report Top-issues section.) - Resolution detection must read each issue's linked/closing PR live merged state (gh pr view ... mergedAt) + the issue state; that live state decides open-vs-resolved, not the thread/body. A MERGED closing PR or CLOSED issue = Resolved. (weekly-report steps 9 + 10; deep-read-issue emits a RESOLVED verdict.) - The final state-refresh before publish covers every referenced item across BOTH repos, never a user-named subset (a headline can close the same day). (weekly-report supervisor rule.) Also: docs/community-signal/reddit-pulse-seen.json gets the 2026-08-07 dedup entry (50 ids) for this week's Reddit Pulse run. --- .claude/skills/deep-read-issue/SKILL.md | 2 +- .claude/skills/weekly-report/SKILL.md | 6 +- docs/community-signal/reddit-pulse-seen.json | 117 +++++++++++++++++-- 3 files changed, 110 insertions(+), 15 deletions(-) diff --git a/.claude/skills/deep-read-issue/SKILL.md b/.claude/skills/deep-read-issue/SKILL.md index bc30c84c..40b535f8 100644 --- a/.claude/skills/deep-read-issue/SKILL.md +++ b/.claude/skills/deep-read-issue/SKILL.md @@ -43,7 +43,7 @@ For EACH issue number, run ALL of these — do not stop at the body: Then deep-read every PR you find (section B) — a fix usually lives here, even when the issue thread is empty. Capture per issue: -- **STATUS AS OF :** one line — open / closed(reason) / fixed-in-vX / merged-PR#N — derived from state + maintainer comment + linked PR, NOT from the body. +- **STATUS AS OF :** one line — open / closed(reason) / fixed-in-vX / merged-PR#N — derived from state + maintainer comment + linked PR, NOT from the body. **If the issue is closed OR its closing/linked PR is MERGED, emit an explicit `RESOLVED` verdict (with PR# + merge/close date)** so the orchestrator files it under ✅ Resolved, never as an open Pain/Top issue. Reading the linked PR's live merged state is a required check — a merged fix outranks any earlier "this is a blocker" comment in the thread. - **TYPE — bug vs feature-request (MANDATORY, run the checks BEFORE calling anything a bug).** Don't assume an issue is a bug. Classify it first, from signals in this order: - **Feature request** if ANY of: the issue title says `Feature Request` / `[Feature]` / `Proposal` / `RFC` / `Enhancement`; the issue has a `feature` / `enhancement` / `proposal` label; **its fix PR is `feat(...)` (conventional-commit) or its title/body says "proposal"** (a `feat` or "proposal" PR is the strongest signal — treat as feature). Feature requests belong in **🔥 Demand**, never in 💢 Pain or 🔝 Top issues (unless a front-door break). - **Bug** if: `[Bug]` title / `bug` label / a `fix(...)` PR / the reporter says something errors, crashes, throws, 404s, regressed, or returns wrong output. diff --git a/.claude/skills/weekly-report/SKILL.md b/.claude/skills/weekly-report/SKILL.md index 6426e073..83c6f3d4 100644 --- a/.claude/skills/weekly-report/SKILL.md +++ b/.claude/skills/weekly-report/SKILL.md @@ -33,6 +33,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl - **Reconcile contradictions between subagents** — if two returns disagree (e.g. deep-read says OPEN but release-scan says shipped), run it down before writing. - **The two formal gates are still mandatory:** the **link-review pass (step 14)** re-verifies every link + product-surface claim, and the **report-sources pass (14b)** defends every placement against evidence and **feeds corrections back into the report** (fix the report first, then the defense reflects it). Loop each until clean. - **Re-spawn or correct** when a return looks off, rather than passing it through. Precedents this cycle: the Fri→Fri window was set to the wrong week and caught mid-run; enrich flagged a stale ("ex-") employer; the release cross-check caught issues already fixed in a shipped release. None of those should reach the published page. +- **The final state-refresh before publish (and any post-publish edit pass) covers EVERY referenced item, BOTH repos — never a subset.** Re-pull the live state of every issue/PR the report cites, even ones nobody flagged. A headline issue can close the same day it's ranked. If the user says "I touched some AG-UI items," still re-verify the CopilotKit items too. (Precedent: [#4893](https://github.com/CopilotKit/CopilotKit/issues/4893) was Top issue #1 at build time and closed hours later; an AG-UI-only refresh missed it because it was CopilotKit — it should have moved to Resolved.) ## Orchestrator flow @@ -99,7 +100,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl - **Reddit mentions / week** — from Subagent E's 90-day pull (≈13 weeks), bucket the surfaced+deduped brand-term posts by `created_utc`. Usually sparse → render as a one-line note, not a weekly bar chart, unless volume justifies bars. - Hand the series to the synthesis as small integer arrays → render as ASCII bars in the collapsible Trends toggle. This week's row is the bottom; the strip is what makes "30 issues" read as up/down/flat and shows whether the backlog is growing. -9. **Detect resolutions.** Classify each in-window CLOSED GitHub issue: `FIX_PR_MERGED` / `BACKFILLED` / `FALSE_POSITIVE` / `DUPLICATE` / `WONT_FIX` / `CLOSED_NO_ACTION`. **Discord threads with a green-check ✅ / accepted-answer marker (step 2) are ALSO resolutions** — classify them `DISCORD_ANSWERED` and list them in ✅ Resolved this week alongside the GitHub closures. (A Discord thread can be resolved even when a related GitHub issue stays open — they're different tickets; resolve only what the green-check actually covers.) +9. **Detect resolutions.** **First, verify live state for EVERY referenced issue — this decides open-vs-resolved, not the issue body or the thread narrative.** For each issue: `gh issue view --repo --json state,stateReason,closedAt` AND follow its linked/closing PRs and check their merged state: `gh pr view --repo --json state,mergedAt,reviewDecision` (or read `closedByPullRequestsReferences`). **If the issue is CLOSED, or its closing PR is MERGED, it is RESOLVED — report it in ✅ Resolved, never as an open Pain/Top issue.** (This is the control that would have caught #4893: its fix PR #5883 was merged and the issue closed; reading the linked PR's merged state is a required check, not optional.) Then classify each in-window CLOSED GitHub issue: `FIX_PR_MERGED` / `BACKFILLED` / `FALSE_POSITIVE` / `DUPLICATE` / `WONT_FIX` / `CLOSED_NO_ACTION`. **Discord threads with a green-check ✅ / accepted-answer marker (step 2) are ALSO resolutions** — classify them `DISCORD_ANSWERED` and list them in ✅ Resolved this week alongside the GitHub closures. (A Discord thread can be resolved even when a related GitHub issue stays open — they're different tickets; resolve only what the green-check actually covers.) 9b. **Cross-check open issues against the release fix-map** (Subagent G). For every issue heading into Demand / Pain / Top issues / Early signals, check the fix-map. If it appears there, it shipped a fix we'd otherwise miss — **flag it inline, in place**: annotate `NOTE: appears fixed in vX.Y.Z (PR #MMMM) — verify` rather than silently reclassifying (a `Fixes #N` in a commit isn't always a complete fix; the human verifies before it moves to Resolved). Also stamp each `✅ Resolved this week` row with its `shipped in vX.Y.Z` from the fix-map. @@ -108,7 +109,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl gh pr list --repo --state all --search "fixes # OR closes #" --json number,title,state,url,isDraft,mergedAt ``` Markers: `🛠️ Fix PR [#NNNN](url) OPEN` · `🛠️ Fix PR [#NNNN](url) MERGED ` · `🛠️ No fix PR yet.` - Procedurally-closed PRs (branch-name violation etc.) don't count as competing fixes — read closing comment. + **The MERGED/OPEN marker must come from the live PR state (`mergedAt` non-null), never inferred from the thread — always read the actual PR.** A **MERGED** closing PR (or a CLOSED issue) means the item is **resolved** → it goes to ✅ Resolved (step 9), not into Pain/Top issues with a "fix PR merged" note. "Fix PR MERGED" on a *still-open* issue is only valid when the merge genuinely didn't resolve it (e.g. partial fix) — say why. Procedurally-closed PRs (branch-name violation etc.) don't count as competing fixes — read the closing comment. 11. **Score & rank the Top issues** (see the ranking rubric in `front-door-triage`). First **record the naive order** — what you'd get ranking the candidates by loudness alone (engagement: 👍 + comments, recency, reporter count) — so the comparison page can show the delta. Then **score each candidate on the five axes** (surface tier · blast radius · severity · exposure · signal), using measurable inputs — `gh issue view --json reactionGroups,comments,labels`, fix-PR status from step 10, Discord distinct-reporter counts, and the enrichment. Sum, sort descending; the top 3–5 are the Top issues, ranked. Keep BOTH the scored table and the naive order — they get published in the ranking + comparison child pages (step 12). Community (CK vs AG-UI) is never an axis. @@ -278,6 +279,7 @@ Sections that do NOT use this card format: ✅ Resolved (XML table), 🟠 Reddit The body **leads** with `## 🔝 Top issues of the week` — cross-community, directly under the CopilotKit header + companion link. Front-door breaks ARE the top issues, ranked together across both repos. What goes in it (per leadership): +- **A merged/confirmed fix is NOT an open Top issue — rank by "is the underlying problem solved?", not "is the ticket open?"** If the fix PR is MERGED (even if unreleased), or a maintainer/reporter says "this can be closed," the item is **resolved / release-pending** → put it in ✅ Resolved (note "shipped-pending-release" if the release hasn't cut), never at the top of the open-break list. Read the thread to the BOTTOM: a "looks fixed on main, can close" comment outranks an earlier "this is a blocker" escalation. (Precedent: [#4893](https://github.com/CopilotKit/CopilotKit/issues/4893) was ranked Top issue #1 as "fix merged, unreleased" when the thread already had a maintainer "can be closed" and it closed that day — it should have been Resolved from the start.) - **Not exhaustive — only what leadership should actually know.** 3–5 items, max. A quiet week can have fewer. - **Ranked by importance.** Number them `### 1.` `### 2.` … Lead with the biggest front-door break — the surface the most users hit. A broken install/quickstart CLI (e.g. `npx create-ag-ui-app`) is a bigger front door than any single feature bug; an outage on the current release is front-page. - **Each card is self-contained** — four lines: diff --git a/docs/community-signal/reddit-pulse-seen.json b/docs/community-signal/reddit-pulse-seen.json index 4baefca1..4c29ff9c 100644 --- a/docs/community-signal/reddit-pulse-seen.json +++ b/docs/community-signal/reddit-pulse-seen.json @@ -1,5 +1,5 @@ { - "_comment": "DEDUP DATABASE for Reddit Pulse — operational state, NOT the report and NOT the Composio/Reddit connection. The Weekly Community Signal report lives ONLY in Notion; this file is the one allowed data artifact in the repo. It records Reddit post IDs already surfaced in past reports so the rolling 90-day Reddit Pulse never re-reports the same post. Each weekly run: read this file, skip any id listed, then append the run's surfaced + noise ids under a new dated entry; prune ids older than 90 days (they can't reappear in the window). NOTE: connecting to Composio/Reddit is separate and lives in .env (COMPOSIO_API_KEY, write-scoped) + a live connected_account_id fetched at runtime — none of that is stored here.", + "_comment": "DEDUP DATABASE for Reddit Pulse \u2014 operational state, NOT the report and NOT the Composio/Reddit connection. The Weekly Community Signal report lives ONLY in Notion; this file is the one allowed data artifact in the repo. It records Reddit post IDs already surfaced in past reports so the rolling 90-day Reddit Pulse never re-reports the same post. Each weekly run: read this file, skip any id listed, then append the run's surfaced + noise ids under a new dated entry; prune ids older than 90 days (they can't reappear in the window). NOTE: connecting to Composio/Reddit is separate and lives in .env (COMPOSIO_API_KEY, write-scoped) + a live connected_account_id fetched at runtime \u2014 none of that is stored here.", "runs": [ { "run_date": "2026-06-19", @@ -84,17 +84,110 @@ "run_date": "2026-07-24", "window": "2026-04-25..2026-07-24", "seen_ids": [ - "1uy96fy", "1uyoahi", "1uyoxt6", "1uyrx7s", "1uz94e7", - "1v2tif9", "1v3bda8", "1v3cuaq", "1v3dixf", "1v3hism", - "1v3itoj", "1v3v4g6", "1v3v50w", "1v3vkm7", "1v3yhdh", - "1v3yy29", "1v44fci", "1v45xq9", "1v47cou", "1v4ajgy", - "1v4crst", "1v4ebjo", "1v4kh48", "1v4l8na", "1v4mods", - "1v4oarx", "1v4q5nq", "1v4w1kt", "1v4wi3y", "1v4zpxt", - "1v53hk1", "1v54xl5", "1v58gll", "1v59yej", "1v5a92k", - "1v5el2x", "1v5eny9", "1v5eos9", "1v5itwd", "1v5j4eb", - "1v5jv69", "1v5jvx1", "1v5jwce", "1v5jwx2", "1v5jxei", - "1v5k28y", "1v5krfv" + "1uy96fy", + "1uyoahi", + "1uyoxt6", + "1uyrx7s", + "1uz94e7", + "1v2tif9", + "1v3bda8", + "1v3cuaq", + "1v3dixf", + "1v3hism", + "1v3itoj", + "1v3v4g6", + "1v3v50w", + "1v3vkm7", + "1v3yhdh", + "1v3yy29", + "1v44fci", + "1v45xq9", + "1v47cou", + "1v4ajgy", + "1v4crst", + "1v4ebjo", + "1v4kh48", + "1v4l8na", + "1v4mods", + "1v4oarx", + "1v4q5nq", + "1v4w1kt", + "1v4wi3y", + "1v4zpxt", + "1v53hk1", + "1v54xl5", + "1v58gll", + "1v59yej", + "1v5a92k", + "1v5el2x", + "1v5eny9", + "1v5eos9", + "1v5itwd", + "1v5j4eb", + "1v5jv69", + "1v5jvx1", + "1v5jwce", + "1v5jwx2", + "1v5jxei", + "1v5k28y", + "1v5krfv" + ] + }, + { + "run_date": "2026-08-07", + "window": "2026-05-09..2026-08-07", + "seen_ids": [ + "1vhwsx1", + "1vharmm", + "1vh778t", + "1vh3kx8", + "1vh33fk", + "1vh32uo", + "1vgfpbq", + "1vg42gr", + "1vg2zu2", + "1vfxrt1", + "1vfxrn4", + "1vfxrlo", + "1vfbflo", + "1vf6lo6", + "1veb1s9", + "1ve8j9i", + "1vdgn6p", + "1vbxgt7", + "1vbxgee", + "1vbxfzf", + "1vbxfi9", + "1vbxedh", + "1vboag1", + "1vbn4bs", + "1vblmhs", + "1vhzj4j", + "1vhygzk", + "1vhudl6", + "1vhud3w", + "1vht0xe", + "1vhmdk7", + "1vhmad3", + "1vhlumc", + "1vhf85p", + "1vhcuua", + "1vh65ub", + "1vh34at", + "1vgz7sz", + "1vgxw59", + "1vgxvp0", + "1vgwqec", + "1vgwhxe", + "1vgs83m", + "1vgppcj", + "1vgp8ml", + "1vgg0cu", + "1vge0xx", + "1vgb7si", + "1vgb3il", + "1vg9wa8" ] } ] -} +} \ No newline at end of file From ad359d96ce85ccc76e75a7c629ca1c8f1341ac5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:59:56 -0400 Subject: [PATCH 54/83] docs(community-signal): enrichment resolves LinkedIn from self-linked accounts first (sales accuracy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A prospect (Ashling Partners' Naveen) was published with a name-searched LinkedIn pointing at a DIFFERENT person, and only his first name — while his GitHub profile self-linked the correct LinkedIn and his blog gave the full name. Root-cause fix so sales links are exact: - enrich-prospect: identity/LinkedIn resolution is now an ordered ladder — (a) self-linked account (gh api users//social_accounts + blog) is authoritative, used verbatim, no search; (b) only if none self-linked, a Google-style web search; (c) match-gate the SEARCHED profile's employer to GitHub company; (d) full name from the authoritative source, not GitHub `name` (often first-name/handle); (e) verify name matches the profile — LinkedIn 999s to fetchers, so confirm via a fetchable self-owned source (blog / authored article byline); (f) else "LinkedIn not confirmed", no guess. - enrich-reporter: also pulls social_accounts, returns the self-linked linkedin_url so the prospect pass reuses it instead of searching. - weekly-report link-review (Subagent F): every prospect LinkedIn is re-verified against gh social_accounts before publish; a differing link is a wrong-person guess and is corrected; the published name must match the profile. Mirrored in the Notion Playbook + Changelog. --- .claude/skills/enrich-prospect/SKILL.md | 19 ++++++++++++------- .claude/skills/enrich-reporter/SKILL.md | 7 +++++-- .claude/skills/weekly-report/SKILL.md | 1 + 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.claude/skills/enrich-prospect/SKILL.md b/.claude/skills/enrich-prospect/SKILL.md index c8d19f40..4d3d4d35 100644 --- a/.claude/skills/enrich-prospect/SKILL.md +++ b/.claude/skills/enrich-prospect/SKILL.md @@ -18,15 +18,18 @@ You are deep-enriching enterprise sales prospects for a community report. Accura For EACH prospect below (handle + the issue/thread that surfaced them): -1. Seed from GitHub: +1. Seed from GitHub — INCLUDING the self-linked accounts: gh api users/ --jq '{login, name, company, bio, blog, twitter_username, email, location}' - Capture their name, the company field, and any employer clue in bio/blog. + gh api users//social_accounts # ← the authoritative self-linked LinkedIn / X / etc. + Capture their name, the company field, employer clues in bio/blog, AND the `social_accounts` URLs. -2. Find their LinkedIn profile: - - WebSearch: " LinkedIn", then " LinkedIn", then " LinkedIn". - - The candidate profile's CURRENT employer MUST match the GitHub company (or a bio/blog employer clue). This is the match gate. - - IF IT DOESN'T MATCH, KEEP SEARCHING — try the blog/personal site, the company team/about page, the Twitter/X bio, a plain Google-style query ("" ""), a GitHub-email search. Only accept a LinkedIn URL when the employer lines up. - - If after real effort you cannot confirm the same person, DO NOT link a guess. Output `Name: — LinkedIn not confirmed` and say what you tried. +2. **Resolve the LinkedIn URL + full name — work this ladder IN ORDER, stop at the first step that yields a confirmed profile:** + a. **Self-linked = authoritative (try FIRST, before any search).** If `social_accounts` has a `linkedin` URL, that IS the profile — use it verbatim, no search, no guessing. Also check the `blog` field (people put their LinkedIn there). A self-linked URL always beats a search hit. *(Miss precedent: the agent name-searched and linked a different person `in/nchatlapalli` when GitHub `social_accounts` already listed `in/navaifanatic`.)* + b. **Only if nothing is self-linked, search — Google-style web search.** Try, in order: `"" "" LinkedIn`, then ` LinkedIn`, then ` LinkedIn`, then a plain `"" ""`. Also try the company team/about page, the X/Twitter bio, and a GitHub-email search. (`WebSearch` is the tool; it's a general web/Google search.) + c. **Match gate — applies to any SEARCHED profile (skip for a self-linked URL, which is already theirs).** Accept a searched profile ONLY when its CURRENT employer matches the GitHub `company` (or a clear bio/blog employer). If it doesn't line up, it's a different person — keep searching. Also apply the stale-employer rule ("ex-"/"previously" ≠ current). + d. **Full name — from the authoritative source, NOT GitHub `name`.** GitHub `name` is often just a first name or a handle. Read the self-linked blog/site (or the LinkedIn) for the complete first + last name. *(Precedent: GitHub `name` was "Naveen"; his blog gave "Naveen Chatlapalli" — a first name alone is a flub on a sales list.)* + e. **Verify the published name matches the profile.** LinkedIn usually returns HTTP 999 to fetchers, so confirm the full name against a *fetchable* self-owned source — their blog, or a LinkedIn article they authored (byline). The name on the report must match the linked profile. + f. **If still unconfirmed after all of the above, DO NOT link a guess.** Output `Name: — LinkedIn not confirmed` and list what you tried, so a human can finish it. 3. Company website: find the official company site (not a directory page). Prefer the root domain. @@ -48,7 +51,9 @@ Return, per prospect, the exact block format in "Output block" below. Cite a rea - **The LinkedIn person's current employer must match the GitHub `company` (or a clear bio/blog employer).** If GitHub says `@commercetools` and the first LinkedIn hit works somewhere else, that's a different person — keep searching. - **Never link a "maybe".** A wrong LinkedIn link in a sales handoff is a real cost. When unconfirmed, write `LinkedIn not confirmed` and list what was tried, so a human can finish it. +- **Use the person's FULL name from the authoritative source** (the self-linked LinkedIn or blog), NOT GitHub's `name` field — that's often just a first name or a handle. If GitHub `name` is partial, read the self-linked blog/site (or the LinkedIn) for the complete first + last name before publishing. Precedent: GitHub `name` was "Naveen"; his self-linked blog gave the full "Naveen Chatlapalli" — publishing just "Naveen" is a flub for a sales list. - **Stale-employer rule (same as enrich-reporter):** if the bio says "ex-", "previously", "formerly", that employer does NOT count as current — it disqualifies both the prospect classification and the match. +- **Don't over-hedge a lead that checks out.** When the self-linked profile, the GitHub `company` field, and a corroborating web search all point to the SAME current employer, mark the prospect **confirmed** — don't leave it "verify before outreach." Reserve `LinkedIn not confirmed` / `employer unconfirmed` for a genuine gap (no self-linked profile AND search can't line the employer up). Precedent: Parker Roan self-linked his LinkedIn, GitHub `company` said Shipt, and a search returned "Software Engineer at Shipt" — three matching signals = confirmed, not a maybe. ## Company size — what counts diff --git a/.claude/skills/enrich-reporter/SKILL.md b/.claude/skills/enrich-reporter/SKILL.md index 91906b16..0f11b00c 100644 --- a/.claude/skills/enrich-reporter/SKILL.md +++ b/.claude/skills/enrich-reporter/SKILL.md @@ -14,11 +14,14 @@ Spawn an `Explore` subagent (or any read-only general-purpose) with this prompt: ## Subagent prompt template ``` -For each GitHub username below, run: +For each GitHub username below, run BOTH: gh api users/ --jq '{login, name, company, bio, blog, twitter_username}' + gh api users//social_accounts # ← the person's OWN self-linked LinkedIn / X / site — authoritative Return a compact one-line-per-user table: - login | name | company | profile_url | company_url | bio | blog | twitter + login | name | company | profile_url | company_url | linkedin_url | bio | blog | twitter + + - linkedin_url = the `linkedin` URL from `social_accounts` if present (the self-linked, authoritative profile — never a search guess); else blank. The prospect pass reuses this instead of searching. - profile_url = `https://github.com/` (always — used to link the handle on every card). - company_url = the company's website for enterprise reporters (e.g. Amazon → https://www.amazon.com, Nvidia → https://www.nvidia.com); blank for indie. Used to link the 🏢 Company badge. Don't guess a URL — leave blank if unsure. diff --git a/.claude/skills/weekly-report/SKILL.md b/.claude/skills/weekly-report/SKILL.md index 83c6f3d4..0f12d78e 100644 --- a/.claude/skills/weekly-report/SKILL.md +++ b/.claude/skills/weekly-report/SKILL.md @@ -131,6 +131,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl 14. **Spawn Subagent F — link review (after the pages are built).** A dedicated review pass over both published pages. Two jobs: - **Coverage — every item has a source link.** Scan every Top-issue card, Demand/Pain bullet, Docs bullet, Resolved row, Reddit Pulse thread, Enterprise reporter, and Patterns entity. **Any item with no source link is flagged.** For each flagged item, hand it to a search retrieval pass (gh search for the issue/PR, Discord `list_forum_threads`/search for the thread, Composio for the Reddit permalink) to find the canonical link. If a link is found → add it. If none can be found → **the item does not stay on the page** (remove it). No bare claims survive. (See "Source links are mandatory".) - **Correctness.** For links that exist: every Discord thread URL's thread ID came from this run's pull (never memory/prior report) and the anchor matches the thread's title; every issue/PR number matches the title quoted next to it; every Reddit permalink is the one returned by Composio this run; external links (YouTube/Loom repro, docs) appear verbatim in the source — never reconstructed; anchor text names what the reader lands on. + - **🎯 Prospect LinkedIn links are sales-critical — VERIFY each against the person's own GitHub `social_accounts`.** For every prospect, run `gh api users//social_accounts`. If the person self-linked a `linkedin` URL there, the report's LinkedIn link **MUST equal it exactly** — a differing link is a wrong-person guess and must be corrected (or set to "LinkedIn not confirmed"). A self-linked account is authoritative; never publish a name-searched LinkedIn when the profile provides its own. (Precedent: the report linked `in/nchatlapalli` for Ashling Partners' Naveen when his GitHub self-linked `in/navaifanatic` — a different person.) **The published name must match the linked profile — verify it.** LinkedIn itself usually can't be fetched (returns HTTP 999), so confirm the person's FULL name (first + last) against a self-owned source that IS fetchable — their self-linked blog/personal site or a LinkedIn article they authored (byline). GitHub's `name` field is often just a first name or a handle — never publish that alone for a sales list. Don't attach a surname the sources don't support. (Precedent: published "Naveen" then a wrong-person link; his self-linked blog gave the full "Naveen Chatlapalli" and his GitHub self-linked the correct profile.) - **Product-surface claims re-verified against the live page.** Re-fetch the relevant page (`WebFetch` /pricing, /product, Intelligence, the products PDF) for every ⚠️ Product surface contradiction card, every 🏢 Enterprise "Surfaces this week" row, and any tier/price/Premium/free/"coming soon" statement anywhere in the report. Confirm the exact claim appears on the live page **now**. Anything that can't be quote-confirmed is **corrected or removed before publish** — a contradiction whose two quotes don't both check out is dropped. Returns: the flagged-item list + what was retrieved/removed. Re-run until zero linkless items remain. From ea05ade792215fe99b96cd7443a3be42aeecfcd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:51:41 -0400 Subject: [PATCH 55/83] =?UTF-8?q?docs(community-signal):=20skill-audit=20c?= =?UTF-8?q?leanup=20=E2=80=94=20dedup=20superseded/duplicated=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two read-only audits (subagent skills + weekly-report orchestrator) found stale and duplicated instructions. Fixes: - weekly-report: Loom framing 5-7 min radio-show -> <=10-min plain briefing (matches loom-walkthrough spec); Reddit render buckets reconciled with the 5 scoring tiers; scoring block relabeled current/v3 (was labeled v2 while carrying v3 values); Early signals added to both page structures (was referenced in flow but absent from the spec); Composio-REST + source-link duplications reduced to cross-refs. - enrich-prospect: Match gate + Company size collapsed to pointers into the step-2 identity ladder / step-4 rule (kept the unique over-hedge calibration rule); description rewritten self-link-first. - slack-tldr: dropped-lines rule de-duped; webhook var aligned to SLACK_WEBHOOK_URL_1. - loom-walkthrough: removed 4th 'never skip AG-UI' repeat + hype-opener repeat; 'cut for 5 min' -> 'under 10 minutes'. --- .claude/skills/enrich-prospect/SKILL.md | 15 ++++----------- .claude/skills/loom-walkthrough/SKILL.md | 6 ++---- .claude/skills/slack-tldr/SKILL.md | 6 +++--- .claude/skills/weekly-report/SKILL.md | 17 +++++++++-------- 4 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.claude/skills/enrich-prospect/SKILL.md b/.claude/skills/enrich-prospect/SKILL.md index 4d3d4d35..fe834eaa 100644 --- a/.claude/skills/enrich-prospect/SKILL.md +++ b/.claude/skills/enrich-prospect/SKILL.md @@ -1,6 +1,6 @@ --- name: enrich-prospect -description: Deep enrichment for community-sourced enterprise PROSPECTS (the 🎯 Prospective enterprise customers subsection). For each prospect, find their LinkedIn profile, verify the LinkedIn employer matches the company on their GitHub (keep searching if it doesn't), then list the company website + company size (ARR / latest funding round / employee count). Runs as a general-purpose subagent (many web searches) so the orchestrator's context stays small. Invoked by weekly-report (enterprise section) and the enterprise skill. +description: Deep enrichment for community-sourced enterprise PROSPECTS (the 🎯 Prospective enterprise customers subsection). For each prospect, resolve their LinkedIn from their GitHub self-linked accounts FIRST (search only if none is self-linked), verify the employer matches, capture their full name, then list the company website + company size (ARR / latest funding round / employee count). Runs as a general-purpose subagent (many web searches) so the orchestrator's context stays small. Invoked by weekly-report (enterprise section) and the enterprise skill. --- # Deep-enrich enterprise prospects @@ -49,20 +49,13 @@ Return, per prospect, the exact block format in "Output block" below. Cite a rea ## Match gate (identity accuracy) -- **The LinkedIn person's current employer must match the GitHub `company` (or a clear bio/blog employer).** If GitHub says `@commercetools` and the first LinkedIn hit works somewhere else, that's a different person — keep searching. -- **Never link a "maybe".** A wrong LinkedIn link in a sales handoff is a real cost. When unconfirmed, write `LinkedIn not confirmed` and list what was tried, so a human can finish it. -- **Use the person's FULL name from the authoritative source** (the self-linked LinkedIn or blog), NOT GitHub's `name` field — that's often just a first name or a handle. If GitHub `name` is partial, read the self-linked blog/site (or the LinkedIn) for the complete first + last name before publishing. Precedent: GitHub `name` was "Naveen"; his self-linked blog gave the full "Naveen Chatlapalli" — publishing just "Naveen" is a flub for a sales list. -- **Stale-employer rule (same as enrich-reporter):** if the bio says "ex-", "previously", "formerly", that employer does NOT count as current — it disqualifies both the prospect classification and the match. +The identity procedure is the ordered ladder in **step 2 of the subagent prompt above** (self-link first → search → match-gate → full name → verify name↔profile → else not-confirmed). Don't restate it here. This section carries only the one calibration rule the ladder doesn't: + - **Don't over-hedge a lead that checks out.** When the self-linked profile, the GitHub `company` field, and a corroborating web search all point to the SAME current employer, mark the prospect **confirmed** — don't leave it "verify before outreach." Reserve `LinkedIn not confirmed` / `employer unconfirmed` for a genuine gap (no self-linked profile AND search can't line the employer up). Precedent: Parker Roan self-linked his LinkedIn, GitHub `company` said Shipt, and a search returned "Software Engineer at Shipt" — three matching signals = confirmed, not a maybe. ## Company size — what counts -Report the single best available signal, most-recent only: - -- **ARR** — only if publicly stated (rare for private co's). -- **Funding — latest round ONLY.** "Series C, $120M, Oct 2024." Do not list the full round history; the current stage is what sales needs. -- **Employees** — a real count or a LinkedIn size band. -- **Unknown** — `size unknown (private, no public figures)`. Honest beats invented. +See **step 4 of the subagent prompt above** for the rule. One signal, most-recent only: ARR (only if publicly stated) → latest funding round ONLY (e.g. "Series C, $120M, Oct 2024", not the full history) → employee count / LinkedIn size band → else `size unknown (private, no public figures)`. Never fabricate a number. ## Output block (the required format) diff --git a/.claude/skills/loom-walkthrough/SKILL.md b/.claude/skills/loom-walkthrough/SKILL.md index 4d37fa46..9fe00ef8 100644 --- a/.claude/skills/loom-walkthrough/SKILL.md +++ b/.claude/skills/loom-walkthrough/SKILL.md @@ -36,7 +36,7 @@ The Top issues are the core of the briefing. Deliver them as a **numbered list m ## Segment flow — walk the report top to bottom -**Dive straight in and follow the report's own order, top to bottom — the CopilotKit page first, then the AG-UI sub-page in the same order.** No separate "opener," no hype line, no curated narrative that reorders the page. One short orienting sentence ("This is the community signal for the week of X — I'll walk CopilotKit top to bottom, then AG-UI"), then go. Cover each section in the order it appears on the page; skip a section only when it's empty (say nothing, move on). +**Dive straight in and follow the report's own order, top to bottom — the CopilotKit page first, then the AG-UI sub-page in the same order.** The page order IS the order — no curated narrative that reorders it (opener/hype rules live in `## Tone`). One short orienting sentence ("This is the community signal for the week of X — I'll walk CopilotKit top to bottom, then AG-UI"), then go. Cover each section in the order it appears on the page; skip a section only when it's empty (say nothing, move on). **CopilotKit page (in page order):** 1. **Trends** — one or two sentences: heavy or quiet week, and are we keeping up. Note capped bulk-close sweeps so the resolved number isn't misread. @@ -53,8 +53,6 @@ Then `[SWITCH to the AG-UI sub-page]` with a plain spoken transition ("now the A **Close** (~15s) — the week in a sentence, the one or two things the host needs the team to action, point to the linked report. Plain sign-off, no flourish. -**Both pages get airtime — never skip AG-UI.** Even a thin AG-UI week gets its walk + the `[SWITCH to the AG-UI sub-page]` cue; compress, don't skip. A script that only walks the CopilotKit page is incomplete. - ## Pain segment (what the CEO wants) Lead with this framing in mind: **fixes are one thing; the *pattern* of pain is where to invest.** Per community, answer "where are people actually struggling?" as a theme, not a ticket list: @@ -77,7 +75,7 @@ Two columns / two blocks so the host can hide the cues: - **The spoken script** — what to say, top to bottom, with `[SCROLL to …]` / `[beat]` / `[~m:ss]` cues in brackets the host reads silently. **Bold the one anchor line per segment** so a host who blanks can just read the bold and move on. Top issues appear as a numbered list, one beat each. - **A cue card** (≤12 lines) — scroll cues + bold anchors only, for off-screen glancing while recording. List the top issues numbered. -- **Pacing notes** — which segment to slow down on (the Pain read), and what to cut for 5 min. No through-line/catchphrase note. +- **Pacing notes** — which segment to slow down on (the Pain read), and which lines to cut to stay under 10 minutes (per the Length section). No through-line/catchphrase note. ## Hand-off diff --git a/.claude/skills/slack-tldr/SKILL.md b/.claude/skills/slack-tldr/SKILL.md index 391091b6..b5db5dfa 100644 --- a/.claude/skills/slack-tldr/SKILL.md +++ b/.claude/skills/slack-tldr/SKILL.md @@ -36,7 +36,7 @@ The Slack app posting this is named `CopilotKit Community Signal` — its name r Full report → <|-
> ``` -This nested layout replaced the older flat single-line Top-issues bullet + the **Top demand** and **🏢 Enterprise reporters** lines (dropped — they cluttered the scan; the full report carries them). Re-add a dropped line only if a week genuinely needs it. +This nested layout replaced the older flat single-line Top-issues bullet. (**Top demand** and **🏢 Enterprise reporters** were also dropped from the TL;DR — see the **Dropped lines** rule below.) ## Rules @@ -72,10 +72,10 @@ Use `\n` for line breaks inside the JSON string. Escape `*` as needed if it appe After saving, output the exact curl Nathan runs: ```bash -curl -X POST -H "Content-Type: application/json" --data @/tmp/slack-msg.json "$SLACK_WEBHOOK_URL" +curl -X POST -H "Content-Type: application/json" --data @/tmp/slack-msg.json "$SLACK_WEBHOOK_URL_1" ``` -Webhook URL lives in Nathan's env. Don't include the URL inline; tell him to `export SLACK_WEBHOOK_URL=...` from the Slack app config first. +Webhook URL lives in Nathan's env as `SLACK_WEBHOOK_URL_1` (set in `.claude/settings.local.json` `env`). Don't include the URL inline; if it isn't already exported, tell him to `export SLACK_WEBHOOK_URL_1=...` from the Slack app config first. ## Dual-community handling diff --git a/.claude/skills/weekly-report/SKILL.md b/.claude/skills/weekly-report/SKILL.md index 0f12d78e..d5869ac3 100644 --- a/.claude/skills/weekly-report/SKILL.md +++ b/.claude/skills/weekly-report/SKILL.md @@ -79,7 +79,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl - `REDDIT_RETRIEVE_POST_COMMENTS` — for high-signal / debatable threads; pass the **bare base36 article id** (no `t3_`). Top comments are the sentiment. - **Relevance filter:** keep only genuine CopilotKit/AG-UI posts. Drop false positives (e.g. the `jscpd` tool listing CopilotKit in a scanned-repo list) and ambiguous `ag-ui` matches — but still record their ids in the ledger. - **Classify per post** 👍 good / 🙂 mixed-positive / 😐 neutral / 🫤 mixed-negative / 👎 pain, from post + top comments. Flag competitor comparisons (LangGraph, Vercel AI SDK, assistant-ui, Vapi…) and recurring comment themes (e.g. "how is AG-UI different from Google A2UI?"). - - **For scoring (v2), fetch each distinct subreddit's recent `new` feed** (`REDDIT_RETRIEVE_REDDIT_POST` sort=new, ~30) → median of `(upvotes + 2·comments)` = the room baseline `M`. Needed for the reach weight + reception ratio (see "Reddit Pulse scoring algorithm"). + - **For scoring, fetch each distinct subreddit's recent `new` feed** (`REDDIT_RETRIEVE_REDDIT_POST` sort=new, ~30) → median of `(upvotes + 2·comments)` = the room baseline `M`. Needed for the reach weight + reception ratio (see "Reddit Pulse scoring algorithm"). - **Split by community subject** (see "Reddit Pulse section") and **score each page 0–100** (see "Reddit Pulse scoring algorithm"). Returns, per community: scored post list (`👍/🙂/😐/🫤/👎 · [title](permalink) · r/ · ⬆score 💬comments · one-line`), the computed Pulse Score + band, an overall-vibe sentence, competitor + recurring-theme notes, and the list of ids to add to the ledger. @@ -139,7 +139,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl **Findings feed BACK into the report — always.** If this pass uncovers a discrepancy (wrong resolved class/date, wrong attribution, stale version, a rank whose inputs don't add up, a "fixed" with no merged PR), **correct the report item first, then the defense reflects the corrected state** — the Report Sources page never sits next to a report it just proved wrong. Loop until zero entries contradict the report. (Precedent: the sources pass caught `ag-ui#2048` listed as `FIX_PR_MERGED / 07-01` when it was `CLOSED COMPLETED 2026-06-29` with no linked PR → the Resolved row was corrected, then defended.) **This pass also writes the `Gaps & follow-ups` items** — it returns a short plain-human checklist of what's unresolved, which the orchestrator drops into the report's Gaps section. Written so the reader can't tell it came from an evidence pass (no lawyer voice, no citations) — see `report-sources`. -15. **Generate the Loom walkthrough script + remind Nathan to record it — every report, no exceptions.** As the LAST step, invoke the `loom-walkthrough` skill to produce the 5–7 min radio-show script from the finished report (plain English, sounds ad-libbed, includes the CEO-level Pain read) so recording is painless. Then remind him to record. When he shares the link: add a `**Loom:** [Walkthrough](url)` line to the main page header (directly under the `**Week:**` line) and a `🎥 Walkthrough → ` line to the Slack message above the "Full report" link. Don't let the Slack message go out without asking about the Loom first. +15. **Generate the Loom walkthrough script + remind Nathan to record it — every report, no exceptions.** As the LAST step, invoke the `loom-walkthrough` skill to produce the ≤10-minute plain-spoken walkthrough briefing from the finished report (plain English, factual — a briefing, not a radio show; includes the CEO-level Pain read) so recording is painless. Then remind him to record. When he shares the link: add a `**Loom:** [Walkthrough](url)` line to the main page header (directly under the `**Week:**` line) and a `🎥 Walkthrough → ` line to the Slack message above the "Full report" link. Don't let the Slack message go out without asking about the Loom first. 16. **Update the ledger + the rules.** Write the run's surfaced + noise post ids into `docs/community-signal/reddit-pulse-seen.json`. And per the meta-rule at the top: if anything about the format changed this run, update these skill files in the same pass. @@ -177,6 +177,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl ### 💢 Pain ← plain `###` header, each item a `#### {toggle}` card (What/Impact/Fix plan) ### 📚 Docs ← standing weekly section; plain `###` header, each item a `#### {toggle}` card (see "Docs section") ### ✅ Resolved this week ← XML table + ### 🌱 Early signals ← CONDITIONAL — `
` block; singletons / one-off low-volume items not yet a pattern (step 7 routes them here). Tables / one-liners, NOT full cards. Omit when there are none. --- ## 🟠 Reddit Pulse — CopilotKit · NN/100 {toggle="true"} ← CopilotKit-subject Reddit posts only, scored. Collapsible; score+band in the heading. (see "Reddit Pulse section") @@ -210,6 +211,7 @@ Covering the **most recent complete Friday→Friday week** (Friday end-date incl ### 🔥 Demand ← plain `###` header, each item a `#### {toggle}` card ### 📚 Docs ← plain `###` header, each item a `#### {toggle}` card ### ✅ Resolved this week + ### 🌱 Early signals ← CONDITIONAL — `
` block; AG-UI singletons not yet a pattern. Tables / one-liners, not full cards. Omit when there are none. --- ## 🟠 Reddit Pulse — AG-UI · NN/100 {toggle="true"} ← AG-UI-subject Reddit posts only, scored. (see "Reddit Pulse section") @@ -272,8 +274,7 @@ Sections that do NOT use this card format: ✅ Resolved (XML table), 🟠 Reddit **Every item on the report carries a source link — no link, it does not get published.** This is a hard rule, not a preference. It applies to every Top-issue card, Demand/Pain bullet, Docs bullet, Resolved row, Reddit Pulse thread, Enterprise reporter, and every named entity in Patterns. - The link points to the **canonical source**: GitHub issue/PR URL, the Discord forum-thread URL, or the Reddit permalink — wherever the claim originated. -- If you have an observation but no sourceable link, **do not write it as a bare claim**. Find the link, or leave it out. A claim with no source is flagged by the review agent (Subagent F, step 14) and either gets a link retrieved or is removed before publish. -- This is what the step-14 review pass enforces: it walks the finished pages, flags every linkless item, retrieves the missing link via search, and deletes anything that still can't be sourced. +- If you have an observation but no sourceable link, **do not write it as a bare claim** — find the link, or leave it out. The step-14 review pass (Subagent F) enforces this: it walks the finished pages, flags every linkless item, retrieves the missing link via search, and deletes anything that still can't be sourced. ## Top issues of the week (the lead body section) @@ -366,7 +367,7 @@ Reddit Pulse is the outside-the-walls read: what people say about CopilotKit / A **Each section is a collapsible toggle** (`## 🟠 Reddit Pulse — · NN/100 {toggle="true"}`) with the 0–100 Pulse Score + band in the heading (so it reads while collapsed). Inside, tab-indented: - the window/dedup note + a one-line link to the algo child page ("How this is scored → 🟠 Reddit Pulse scoring algorithm"); - a **Vibe** sentence (overall sentiment); -- scored post groups (**👍 Good** / **😐 Neutral / awareness** / **👎 Pain**), each post one bullet: `[title](permalink) — r/ ⬆score 💬comments. one-line.`; +- scored post groups — three render buckets: **👍 Good** / **😐 Neutral / awareness** / **👎 Pain**. The scoring step classifies into five sentiment tiers (see the algorithm's `s` values); for display, **🙂 mixed-positive folds into 👍 Good and 🫤 mixed-negative into 👎 Pain**. Each post is one bullet: `[title](permalink) — r/ ⬆score 💬comments. one-line.`; - a **🔁 Recurring** line for comment themes worth addressing (e.g. the A2UI-vs-AG-UI confusion); - a closing `*Net: …· Pulse Score NN/100.*` tally. @@ -375,13 +376,13 @@ Rules: - **Cross-posts** of the same story merge into one bullet (note the copies + use max engagement). - **Noise** (spam, false-positive keyword hits) is dropped from the section but still recorded in the ledger so it can't resurface. - **Source-gated:** if there's no write-scoped `COMPOSIO_API_KEY` or no ACTIVE Reddit connected account, render "🟠 Reddit Pulse — source not configured this week." and move on — never block the report on it. -- **Data source:** **Composio REST** (Composio's egress reaches Reddit where this machine's IP is 403-blocked on anonymous reads) — NOT the `composio` MCP (its OAuth identity can't see the dashboard connection) and NOT a default read-only API key (`tool_execution` 403). Use a **write-scoped** Composio API key (`Tools` resource = Write) in `COMPOSIO_API_KEY` (repo-root `.env`); `POST /api/v3/tools/execute/` with the ACTIVE Reddit `connected_account_id` from `GET /api/v3/connected_accounts?toolkit_slugs=reddit`. Tools: `REDDIT_SEARCH_ACROSS_SUBREDDITS`, `REDDIT_RETRIEVE_REDDIT_POST`, `REDDIT_RETRIEVE_POST_COMMENTS`. Scope vars `REDDIT_BRAND_TERMS` + `REDDIT_WATCHLIST` in the repo-root `.env`. +- **Data source:** **Composio REST** with a **write-scoped** Composio API key (`Tools` resource = Write) in `COMPOSIO_API_KEY` (repo-root `.env`) — see **step 6** for the auth rationale (why not the `composio` MCP, why a read-only key 403s). Call `POST /api/v3/tools/execute/` with the ACTIVE Reddit `connected_account_id` from `GET /api/v3/connected_accounts?toolkit_slugs=reddit`. Tools: `REDDIT_SEARCH_ACROSS_SUBREDDITS`, `REDDIT_RETRIEVE_REDDIT_POST`, `REDDIT_RETRIEVE_POST_COMMENTS`. Scope vars `REDDIT_BRAND_TERMS` + `REDDIT_WATCHLIST` in the repo-root `.env`. ### Reddit Pulse scoring algorithm Each section's 0–100 score is **calculated, not asserted**, and published on a standing public child page (`🟠 Reddit Pulse — scoring algorithm`) linked from each section. Community is never an input — each community is scored on its own posts. -**v2 (2026-06-19) — reach × reception.** v1 weighted by the post's own engagement only, so a win in a tiny sub outweighed a flop in a big one. v2 weights by the *room* and judges each post against that room's own norm: +**Current formula — reach × reception** (introduced as v2 on 2026-06-19, corrected by v3 below; the values in this block are the current v3 values). v1 weighted by the post's own engagement only, so a win in a tiny sub outweighed a flop in a big one. This weights by the *room* and judges each post against that room's own norm: - **Sentiment per post** `s` (from post + top comments): `+1` good · `+0.5` mixed-positive · `0` neutral · `−0.5` mixed-negative · `−1` pain. - **Room baseline** `M` = median of `(upvotes + 2·comments)` over the subreddit's recent **`new`** posts (NOT `hot` — hot oversamples winners). Fetch ~30 per distinct sub. `M` is the room's activity proxy (quiet "<10 posts/day" sub → low `M`). @@ -509,7 +510,7 @@ Test before publishing: read each parenthetical aloud and ask "would a non-engin - `enrich-reporter` — subagent for GitHub author enterprise enrichment (shallow: company field → 🏢 badge, all reporters) - `enrich-prospect` — subagent for DEEP enterprise-prospect enrichment (LinkedIn + company website + size; prospect shortlist only) - `slack-tldr` — Slack JSON format + curl command -- `loom-walkthrough` — the 5–7 min radio-show walkthrough script, generated after every report (last step) +- `loom-walkthrough` — the ≤10-minute plain-spoken walkthrough briefing, generated after every report (last step) - `report-sources` — subagent: the "Report Sources" child page defending every placement with evidence (front-door, rank, section, attribution, resolved, enterprise, maturity) - `enterprise` — standalone enterprise view (run separately or invoked here) - `topic-search` — ad-hoc cross-repo topic lookup From e415e314bb49310c314ede859e479c6bf0948362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:11:15 -0400 Subject: [PATCH 56/83] fix(ai): answer each ticket once, and stop leaking ticket IDs to reporters Outpost was replying to every message in a support thread. In the Discord thread 1535447155735789708 a maintainer posted the real solution and the bot answered 14 seconds later, summarising that answer back at the thread. In thread 1531971013791711342 the bot answered a maintainer's question to a community member. The agent is a first line of defence; a human owns the thread from the first response onward. The invariant is now: exactly one AI response per ticket, on the message that opened it, with no follow-up to any later message regardless of sender -- original reporter, third party, or team member. Team-member detection is irrelevant to this decision and no longer gates it. Enforced in two places. The AI_RESPONSE handler drops any job for a ticket that already carries an AI-authored BOT message; because it reads the ticket's own history rather than trusting its caller, a future enqueue site cannot reintroduce the behaviour. Reply paths additionally stop enqueuing, so a job that would be dropped is never paid for: - packages/outpost/shared/src/platforms/inbound.ts (Discord, Slack, Teams, and GitHub issue bodies -- status transitions retained) - apps/github-app/src/webhooks/issue-comment.ts (its own path, which bypassed InboundHandler entirely) - apps/web/src/app/api/webhooks/postmark/route.ts (email replies) - apps/discord-bot/src/lib/shadow-mode.ts (so shadow mirrors production) Second, internal ticket displayIds no longer reach reporters. The Discord and Slack "Ticket TKT-XXXXXXXX created" acknowledgment posts are removed outright -- they published an internal identifier into a public server and spent a bot message on nothing the reporter could act on. The three Teams Adaptive Cards drop displayId from their text; buildResponseCard keeps it in the Action.Submit data payloads so button clicks still resolve to a ticket. displayId remains in the dashboard, in team slash commands, and in logs. Call-site enumeration: - buildTicketCreatedCard (option removed: ticketDisplayId) -- 1 call site, apps/teams-bot/src/handlers/message.ts:82, updated. Tests updated. No remaining references to the removed option. - buildEscalationCard (option removed: ticketDisplayId) -- 1 call site, apps/teams-bot/src/handlers/card-actions.ts:106, updated. Tests updated. No remaining references to the removed option. - buildResponseCard (signature unchanged; body text changed) -- 1 call site, apps/teams-bot/src/lib/teams-poster.ts:23. Assumption still holds: it passes ticketDisplayId, which is still consumed, now only for action routing. PlatformTeamsAdapter has a separate private buildResponseCard (packages/outpost/shared/src/platforms/teams.ts:269) that never rendered a displayId -- unaffected. - InboundResult.aiJobEnqueued (semantics changed: always false on replies) -- 1 consumer, apps/discord-bot/src/events/message-create.ts:68, which only appends a log suffix. Assumption still holds; the log is now accurate rather than misleading. - handleAiResponse, handleShadowMessage, handleReply -- signatures unchanged; handleReply is private with no external callers. Tests: 1778 pass across all 10 packages. 11 existing tests asserted the old behaviour and were rewritten -- notably the github-app and discord-bot cases that locked in "answer every follow-up". Six new tests cover the handler guard, including that a human reply after the AI response still does not trigger a second answer, that a first response is not swallowed, and that a SYSTEM shadow-mode log is not mistaken for the ticket's answer. Red-green verified: neutralising the guard turns them red. The guard tests mock the AI pipeline at the class seam rather than driving LLMock, because the assertion they exist to make is that no model call happens at all -- generateSupportResponse not being called is the direct expression of that. Not addressed here: the response template claims "we've escalated this to our engineering team" without anything enforcing it. Wiring the copy to real escalation is deferred by request. --- .../src/__tests__/message-create.test.ts | 16 +- .../src/__tests__/shadow-mode.test.ts | 14 +- .../src/__tests__/thread-create.test.ts | 20 +++ apps/discord-bot/src/events/thread-create.ts | 11 +- apps/discord-bot/src/lib/shadow-mode.ts | 10 +- .../src/__tests__/issue-comment.test.ts | 14 +- apps/github-app/src/webhooks/issue-comment.ts | 11 +- .../src/__tests__/inbound-handler.test.ts | 11 +- apps/slack-bot/src/__tests__/message.test.ts | 25 +--- apps/slack-bot/src/events/message.ts | 19 +-- apps/teams-bot/src/__tests__/cards.test.ts | 49 ++++++- .../src/__tests__/inbound-handler.test.ts | 9 +- apps/teams-bot/src/__tests__/message.test.ts | 11 +- apps/teams-bot/src/cards/escalation-card.ts | 8 +- apps/teams-bot/src/cards/response-card.ts | 7 +- .../src/cards/ticket-created-card.ts | 8 +- apps/teams-bot/src/handlers/card-actions.ts | 1 - apps/teams-bot/src/handlers/message.ts | 1 - .../src/__tests__/postmark-webhook.test.ts | 9 +- .../src/app/api/webhooks/postmark/route.ts | 5 +- .../queue/src/__tests__/ai-response.test.ts | 138 ++++++++++++++++++ .../outpost/queue/src/handlers/ai-response.ts | 33 +++++ .../src/__tests__/inbound-handler.test.ts | 11 +- .../src/__tests__/platforms-inbound.test.ts | 88 +++++------ .../outpost/shared/src/platforms/inbound.ts | 47 +++--- 25 files changed, 379 insertions(+), 197 deletions(-) diff --git a/apps/discord-bot/src/__tests__/message-create.test.ts b/apps/discord-bot/src/__tests__/message-create.test.ts index 59c706d0..39142554 100644 --- a/apps/discord-bot/src/__tests__/message-create.test.ts +++ b/apps/discord-bot/src/__tests__/message-create.test.ts @@ -102,7 +102,12 @@ describe('handleMessageCreate', () => { expect(prisma.message.create).not.toHaveBeenCalled(); }); - it('processes reply through InboundHandler and enqueues AI response for non-team-member', async () => { + // ONE RESPONSE PER TICKET. The thread starter gets an answer (see + // thread-create.test.ts); replies in that thread never do, whoever sends + // them. This test used to assert the opposite — it locked in the behaviour + // where the bot answered follow-up messages, including a maintainer's own + // reply in a Discord support thread. + it('appends a reply through InboundHandler without enqueuing an AI response', async () => { const message = makeMessage(); await handleMessageCreate(message); @@ -114,14 +119,7 @@ describe('handleMessageCreate', () => { }), }); - // InboundHandler enqueues AI response via createJob wrapper - expect(createJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - expect.objectContaining({ - ticketId: 'ticket-1', - source: 'discord', - }), - ); + expect(createJob).not.toHaveBeenCalled(); }); it('does not enqueue AI response for team member messages', async () => { diff --git a/apps/discord-bot/src/__tests__/shadow-mode.test.ts b/apps/discord-bot/src/__tests__/shadow-mode.test.ts index 2ad33b7e..d3573d2b 100644 --- a/apps/discord-bot/src/__tests__/shadow-mode.test.ts +++ b/apps/discord-bot/src/__tests__/shadow-mode.test.ts @@ -230,18 +230,14 @@ describe('shadow-mode', () => { }); }); - it('enqueues an AI response job for the ticket', async () => { + // Shadow mode has to mirror production, and production answers a ticket + // once — on its opening message. Enqueuing on replies here would make + // shadow traffic look chattier than the real bot. + it('does not enqueue an AI response job for a reply', async () => { const message = makeMessage(); await handleShadowMessage(message, 'ticket-1', 'thread-123'); - expect(createJob).toHaveBeenCalledWith( - JobType.AI_RESPONSE, - expect.objectContaining({ - ticketId: 'ticket-1', - threadId: 'thread-123', - source: 'discord', - }), - ); + expect(createJob).not.toHaveBeenCalled(); }); }); }); diff --git a/apps/discord-bot/src/__tests__/thread-create.test.ts b/apps/discord-bot/src/__tests__/thread-create.test.ts index 9028bbc3..1d2a81a3 100644 --- a/apps/discord-bot/src/__tests__/thread-create.test.ts +++ b/apps/discord-bot/src/__tests__/thread-create.test.ts @@ -36,6 +36,7 @@ vi.mock('discord.js', async (importOriginal) => { import { handleThreadCreate } from '../events/thread-create.js'; import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; +import { PlatformDiscordAdapter } from '@copilotkit/outpost/shared/platforms'; function makeThread(overrides: Record = {}) { return { @@ -127,6 +128,25 @@ describe('handleThreadCreate', () => { ); }); + // The bot used to open every thread with "🎫 Ticket TKT-XXXXXXXX created…", + // publishing an internal identifier into a public server and spending a bot + // message on nothing the reporter can act on. The AI answer is the only + // message the bot sends. + it('posts no acknowledgment message and never emits the ticket displayId', async () => { + const postSystemMessage = vi.spyOn( + PlatformDiscordAdapter.prototype, + 'postSystemMessage', + ); + + const thread = makeThread(); + await handleThreadCreate(thread, true); + + expect(postSystemMessage).not.toHaveBeenCalled(); + expect(thread.send).not.toHaveBeenCalled(); + + postSystemMessage.mockRestore(); + }); + it('handles threads with no starter message content gracefully', async () => { const thread = makeThread(); vi.mocked(thread.fetchStarterMessage).mockResolvedValue(null); diff --git a/apps/discord-bot/src/events/thread-create.ts b/apps/discord-bot/src/events/thread-create.ts index 83f87139..6006f032 100644 --- a/apps/discord-bot/src/events/thread-create.ts +++ b/apps/discord-bot/src/events/thread-create.ts @@ -81,11 +81,12 @@ export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: bo const handler = new InboundHandler({ prisma, createJob: createJobFn }); const result = await handler.handle(inboundMessage); - // Post acknowledgment in the thread (Discord-specific UX) - await adapter.postSystemMessage( - { id: result.ticketId, sourceId: thread.id, channel: parentId, source: adapter.platform }, - `\uD83C\uDFAB Ticket ${result.displayId} created. Our AI assistant is reviewing your question...`, - ); + // No acknowledgment post. This used to announce + // "\uD83C\uDFAB Ticket TKT-XXXXXXXX created..." in the thread, which leaked an + // internal identifier to the public server and spent a bot message + // saying nothing the reporter can act on. displayId is for the dashboard + // and team slash commands only \u2014 never for reporter-facing copy. + // The AI response itself is the only message the reporter needs. console.log(`[Discord Bot] Created ticket ${result.displayId} for thread ${thread.id}`); } catch (error) { diff --git a/apps/discord-bot/src/lib/shadow-mode.ts b/apps/discord-bot/src/lib/shadow-mode.ts index a1f716cb..5a0763e0 100644 --- a/apps/discord-bot/src/lib/shadow-mode.ts +++ b/apps/discord-bot/src/lib/shadow-mode.ts @@ -132,12 +132,10 @@ export async function handleShadowMessage( }, }); - // Enqueue AI response (shadow mode checked at handler level via SHADOW_MODE env) - await createJob(JobType.AI_RESPONSE, { - ticketId, - threadId, - source: 'discord' as const, - }); + // No AI response on a reply — shadow mode mirrors production behaviour, + // and production answers a ticket once, on its opening message only. + // Enqueuing here would make shadow traffic look chattier than the real + // thing, which defeats the point of shadowing. console.log( `[Shadow Mode] Recorded message from ${message.author.tag} on ticket ${ticketId}`, diff --git a/apps/github-app/src/__tests__/issue-comment.test.ts b/apps/github-app/src/__tests__/issue-comment.test.ts index cf49d6a8..d0c83225 100644 --- a/apps/github-app/src/__tests__/issue-comment.test.ts +++ b/apps/github-app/src/__tests__/issue-comment.test.ts @@ -147,7 +147,11 @@ describe('handleIssueComment', () => { }); }); - it('appends a message and enqueues AI response for non-team-member', async () => { + // ONE RESPONSE PER TICKET. Outpost answers the issue body and then stays out + // of the comment thread — including when the original reporter follows up. + // This used to assert the opposite, locking in the behaviour where the bot + // kept commenting on issues a human had already taken over. + it('appends a comment without enqueuing an AI response for non-team-member', async () => { const event = makeEvent(); await handleIssueComment(event); @@ -159,13 +163,7 @@ describe('handleIssueComment', () => { }), }); - expect(createJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - expect.objectContaining({ - ticketId: 'ticket-1', - source: 'github', - }), - ); + expect(createJob).not.toHaveBeenCalled(); }); it('does not enqueue AI response for team member comments (static list)', async () => { diff --git a/apps/github-app/src/webhooks/issue-comment.ts b/apps/github-app/src/webhooks/issue-comment.ts index fe1697e4..cf176dc2 100644 --- a/apps/github-app/src/webhooks/issue-comment.ts +++ b/apps/github-app/src/webhooks/issue-comment.ts @@ -1,6 +1,5 @@ import type { EmitterWebhookEvent } from '@octokit/webhooks'; import { prisma } from '@copilotkit/outpost/db'; -import { createJob } from '@copilotkit/outpost/queue'; import { GitHubPlatformAdapter } from '@copilotkit/outpost/shared/platforms'; import { getOctokit } from '../lib/github-client.js'; import { findTicketBySourceId, isTeamMember } from '../lib/tickets.js'; @@ -77,11 +76,11 @@ export async function handleIssueComment( }); } } else { - // External user (likely original poster): enqueue AI response - await createJob('AI_RESPONSE' as Parameters[0], { - ticketId: ticket.id, - source: 'github' as const, - }); + // No AI response on comments — Outpost answers the issue body once and + // then stays out of the thread, whoever comments next. This previously + // enqueued an AI_RESPONSE for every non-team commenter, so the bot kept + // replying to follow-ups on issues a human had already picked up. + // The AI_RESPONSE handler enforces the same invariant server-side. // Reopen ticket if it was waiting on customer or resolved if (ticket.status === 'WAITING_ON_CUSTOMER' || ticket.status === 'RESOLVED') { diff --git a/apps/slack-bot/src/__tests__/inbound-handler.test.ts b/apps/slack-bot/src/__tests__/inbound-handler.test.ts index 0dbf4c35..5b44ff91 100644 --- a/apps/slack-bot/src/__tests__/inbound-handler.test.ts +++ b/apps/slack-bot/src/__tests__/inbound-handler.test.ts @@ -139,7 +139,8 @@ describe('InboundHandler (Slack-focused)', () => { vi.mocked(mockPrisma.ticket.findFirst).mockResolvedValue(TICKET); }); - it('appends a message and enqueues AI response for non-team-members', async () => { + // One response per ticket — a thread reply never gets its own AI answer. + it('appends a message without enqueuing an AI response for non-team-members', async () => { const result = await handler.handle(makeThreadReply()); expect(result.isNewTicket).toBe(false); @@ -153,13 +154,7 @@ describe('InboundHandler (Slack-focused)', () => { }), }); - expect(mockCreateJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - expect.objectContaining({ - ticketId: 'ticket-1', - source: 'slack', - }), - ); + expect(mockCreateJob).not.toHaveBeenCalled(); }); it('reopens RESOLVED ticket when customer replies', async () => { diff --git a/apps/slack-bot/src/__tests__/message.test.ts b/apps/slack-bot/src/__tests__/message.test.ts index feb0ac73..bb29fa0f 100644 --- a/apps/slack-bot/src/__tests__/message.test.ts +++ b/apps/slack-bot/src/__tests__/message.test.ts @@ -133,15 +133,11 @@ describe('registerMessageHandler', () => { }), ); - // SlackAdapter posts acknowledgment via postSystemMessage. - // The ticket ID is generated at runtime so we match the pattern. - expect(mockPostMessage).toHaveBeenCalledWith( - expect.objectContaining({ - channel: 'C_MONITORED', - thread_ts: '1234567890.123456', - text: expect.stringMatching(/TKT-[A-Z0-9]+ created/), - }), - ); + // No acknowledgment post. It used to announce "TKT-XXXXXXXX created" + // in-channel, leaking an internal identifier to the reporter and + // spending an extra bot message. The AI response is the only message + // the bot sends. + expect(mockPostMessage).not.toHaveBeenCalled(); }); it('ignores messages in unmonitored channels', async () => { @@ -193,7 +189,8 @@ describe('registerMessageHandler', () => { ); }); - it('appends a message and enqueues AI response for non-team-member replies', async () => { + // One response per ticket — thread replies are recorded, never answered. + it('appends a message without enqueuing an AI response for non-team-member replies', async () => { await messageHandler({ event: { user: 'U_EXTERNAL', @@ -212,13 +209,7 @@ describe('registerMessageHandler', () => { }), }); - expect(createJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - expect.objectContaining({ - ticketId: 'ticket-1', - source: 'slack', - }), - ); + expect(createJob).not.toHaveBeenCalled(); }); it('does not enqueue AI response for team member replies', async () => { diff --git a/apps/slack-bot/src/events/message.ts b/apps/slack-bot/src/events/message.ts index 1996abb4..db5b25d6 100644 --- a/apps/slack-bot/src/events/message.ts +++ b/apps/slack-bot/src/events/message.ts @@ -3,7 +3,6 @@ import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; import { SlackAdapter, InboundHandler } from '@copilotkit/outpost/shared/platforms'; import type { InboundPrismaLike, CreateJobFn } from '@copilotkit/outpost/shared'; -import { TicketSource } from '@copilotkit/outpost/shared'; import { config } from '../config.js'; /** @@ -49,21 +48,11 @@ export function registerMessageHandler(app: App): void { if (!existingTicket) return; } - const result = await handler.handle(message); + await handler.handle(message); - // Post acknowledgment for newly created tickets - if (result.isNewTicket && result.displayId) { - const ticket = { - id: result.ticketId, - sourceId: message.threadId ? `${message.channelId}:${message.threadId}` : null, - channel: message.channelId ?? null, - source: TicketSource.SLACK, - }; - await adapter.postSystemMessage( - ticket, - `\uD83C\uDFAB Ticket ${result.displayId} created. Our AI assistant is reviewing your question...`, - ); - } + // No acknowledgment post \u2014 it leaked the internal ticket displayId to + // the channel and added a second bot message for no reporter benefit. + // See the matching change in apps/discord-bot/src/events/thread-create.ts. } catch (error) { console.error( `[Slack Bot] Failed to process message in channel ${(event as { channel?: string }).channel ?? 'unknown'}:`, diff --git a/apps/teams-bot/src/__tests__/cards.test.ts b/apps/teams-bot/src/__tests__/cards.test.ts index b3b2881c..1a863d87 100644 --- a/apps/teams-bot/src/__tests__/cards.test.ts +++ b/apps/teams-bot/src/__tests__/cards.test.ts @@ -3,6 +3,15 @@ import { buildResponseCard } from '../cards/response-card.js'; import { buildTicketCreatedCard } from '../cards/ticket-created-card.js'; import { buildEscalationCard } from '../cards/escalation-card.js'; +/** + * Every text block in a card body, flattened. Used by the leak assertions + * below so a displayId can't reappear in a block index nobody checks. + */ +function allCardText(card: Record): string { + const body = card.body as Array<{ text?: string }>; + return body.map((b) => b.text ?? '').join('\n'); +} + describe('buildResponseCard', () => { it('builds a card with action buttons', () => { const card = buildResponseCard({ @@ -15,7 +24,7 @@ describe('buildResponseCard', () => { expect(card.version).toBe('1.4'); const body = card.body as Array<{ text: string }>; - expect(body[0].text).toContain('TKT-AB12'); + expect(body[0].text).toBe('AI Response'); expect(body[1].text).toBe('Here is the answer.'); const actions = card.actions as Array<{ title: string; data: { action: string } }>; @@ -26,6 +35,28 @@ describe('buildResponseCard', () => { expect(actions[1].data.action).toBe('need_more_help'); }); + it('never renders the internal ticket displayId into card text', () => { + const card = buildResponseCard({ + ticketDisplayId: 'TKT-AB12', + responseText: 'Here is the answer.', + confidence: 0.9, + }); + + expect(allCardText(card)).not.toContain('TKT-AB12'); + }); + + it('still routes the displayId through action data so clicks resolve', () => { + const card = buildResponseCard({ + ticketDisplayId: 'TKT-AB12', + responseText: 'Here is the answer.', + confidence: 0.9, + }); + + const actions = card.actions as Array<{ data: { ticketDisplayId: string } }>; + expect(actions[0].data.ticketDisplayId).toBe('TKT-AB12'); + expect(actions[1].data.ticketDisplayId).toBe('TKT-AB12'); + }); + it('includes a low-confidence disclaimer when confidence is below threshold', () => { const card = buildResponseCard({ ticketDisplayId: 'TKT-AB12', @@ -53,29 +84,37 @@ describe('buildResponseCard', () => { describe('buildTicketCreatedCard', () => { it('builds a ticket acknowledgment card', () => { const card = buildTicketCreatedCard({ - ticketDisplayId: 'TKT-CD34', title: 'Help with integration', }); expect(card.type).toBe('AdaptiveCard'); const body = card.body as Array<{ text: string }>; - expect(body[0].text).toContain('TKT-CD34'); expect(body[1].text).toBe('Help with integration'); expect(body[2].text).toContain('AI assistant'); }); + + it('never renders a ticket displayId — the option does not exist', () => { + const card = buildTicketCreatedCard({ title: 'Help with integration' }); + + expect(allCardText(card)).not.toMatch(/TKT-/); + }); }); describe('buildEscalationCard', () => { it('builds an escalation notification card', () => { const card = buildEscalationCard({ - ticketDisplayId: 'TKT-EF56', reason: 'User needs more help.', }); expect(card.type).toBe('AdaptiveCard'); const body = card.body as Array<{ text: string }>; - expect(body[0].text).toContain('TKT-EF56'); expect(body[1].text).toBe('User needs more help.'); expect(body[2].text).toContain('team member'); }); + + it('never renders a ticket displayId — the option does not exist', () => { + const card = buildEscalationCard({ reason: 'User needs more help.' }); + + expect(allCardText(card)).not.toMatch(/TKT-/); + }); }); diff --git a/apps/teams-bot/src/__tests__/inbound-handler.test.ts b/apps/teams-bot/src/__tests__/inbound-handler.test.ts index b45702d9..4f124e31 100644 --- a/apps/teams-bot/src/__tests__/inbound-handler.test.ts +++ b/apps/teams-bot/src/__tests__/inbound-handler.test.ts @@ -150,14 +150,11 @@ describe('InboundHandler (Teams-focused)', () => { }); }); - it('enqueues AI response for non-team-member messages', async () => { + // One response per ticket — follow-up messages are recorded, not answered. + it('does not enqueue an AI response for non-team-member follow-ups', async () => { await handler.handle(makeMessage({ isThreadStart: false })); - expect(createJob).toHaveBeenCalledWith('AI_RESPONSE', { - ticketId: 'existing-ticket-id', - threadId: 'conv-100', - source: 'teams', - }); + expect(createJob).not.toHaveBeenCalled(); }); it('transitions WAITING_ON_TEAM to WAITING_ON_CUSTOMER for team member messages', async () => { diff --git a/apps/teams-bot/src/__tests__/message.test.ts b/apps/teams-bot/src/__tests__/message.test.ts index 710191a8..7f17733f 100644 --- a/apps/teams-bot/src/__tests__/message.test.ts +++ b/apps/teams-bot/src/__tests__/message.test.ts @@ -193,14 +193,9 @@ describe('handleMessage', () => { }), }); - // Should enqueue AI response (not a team member) - expect(createJob).toHaveBeenCalledWith( - JobType.AI_RESPONSE, - expect.objectContaining({ - ticketId: 'ticket-1', - source: 'teams', - }), - ); + // Should NOT enqueue an AI response — one answer per ticket, on the + // opening message only, whoever sends the follow-up. + expect(createJob).not.toHaveBeenCalled(); }); it('does not enqueue AI response for team member follow-ups', async () => { diff --git a/apps/teams-bot/src/cards/escalation-card.ts b/apps/teams-bot/src/cards/escalation-card.ts index 8f2d1b32..7499fe30 100644 --- a/apps/teams-bot/src/cards/escalation-card.ts +++ b/apps/teams-bot/src/cards/escalation-card.ts @@ -1,13 +1,15 @@ export interface EscalationCardOptions { - ticketDisplayId: string; reason: string; } /** * Build an Adaptive Card for escalation notifications. + * + * Carries no ticket displayId \u2014 this card is shown to the reporter, and the + * identifier is internal to the dashboard and team slash commands. */ export function buildEscalationCard(options: EscalationCardOptions): Record { - const { ticketDisplayId, reason } = options; + const { reason } = options; return { type: 'AdaptiveCard', @@ -16,7 +18,7 @@ export function buildEscalationCard(options: EscalationCardOptions): Record[] = [ { type: 'TextBlock', - text: `**${ticketDisplayId}** - AI Response`, + text: 'AI Response', weight: 'Bolder', size: 'Medium', }, diff --git a/apps/teams-bot/src/cards/ticket-created-card.ts b/apps/teams-bot/src/cards/ticket-created-card.ts index d2a9f619..fa1bc66c 100644 --- a/apps/teams-bot/src/cards/ticket-created-card.ts +++ b/apps/teams-bot/src/cards/ticket-created-card.ts @@ -1,13 +1,15 @@ export interface TicketCreatedCardOptions { - ticketDisplayId: string; title: string; } /** * Build an Adaptive Card acknowledging ticket creation. + * + * Deliberately carries no ticket displayId. That identifier is internal \u2014 it + * belongs in the dashboard and team slash commands, not in reporter-facing copy. */ export function buildTicketCreatedCard(options: TicketCreatedCardOptions): Record { - const { ticketDisplayId, title } = options; + const { title } = options; return { type: 'AdaptiveCard', @@ -16,7 +18,7 @@ export function buildTicketCreatedCard(options: TicketCreatedCardOptions): Recor body: [ { type: 'TextBlock', - text: `\uD83C\uDFAB Ticket ${ticketDisplayId} created`, + text: "\uD83C\uDFAB We've got your question", weight: 'Bolder', size: 'Medium', }, diff --git a/apps/teams-bot/src/handlers/card-actions.ts b/apps/teams-bot/src/handlers/card-actions.ts index c29ba15c..c3978409 100644 --- a/apps/teams-bot/src/handlers/card-actions.ts +++ b/apps/teams-bot/src/handlers/card-actions.ts @@ -104,7 +104,6 @@ async function handleNeedMoreHelp(context: TurnContext, _data: CardActionData): // Post escalation card const card = buildEscalationCard({ - ticketDisplayId: ticket.displayId, reason: 'User requested additional assistance.', }); diff --git a/apps/teams-bot/src/handlers/message.ts b/apps/teams-bot/src/handlers/message.ts index 7cf8df40..02713fc0 100644 --- a/apps/teams-bot/src/handlers/message.ts +++ b/apps/teams-bot/src/handlers/message.ts @@ -80,7 +80,6 @@ export async function handleMessage(context: TurnContext): Promise { // New ticket: post acknowledgment card const card = buildTicketCreatedCard({ - ticketDisplayId: result.displayId, title: truncate(message.content, 200), }); diff --git a/apps/web/src/__tests__/postmark-webhook.test.ts b/apps/web/src/__tests__/postmark-webhook.test.ts index 170842d3..4eea1376 100644 --- a/apps/web/src/__tests__/postmark-webhook.test.ts +++ b/apps/web/src/__tests__/postmark-webhook.test.ts @@ -193,11 +193,10 @@ describe('Postmark inbound webhook', () => { // Should NOT create a new ticket expect(mockTicketCreate).not.toHaveBeenCalled(); - // Should enqueue AI_RESPONSE job for the reply - expect(mockCreateJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - { ticketId: 'existing-ticket-1', source: 'web' }, - ); + // Should NOT enqueue AI_RESPONSE for the reply — one answer per + // ticket, on the opening email only. A human owns the thread after + // the first response. + expect(mockCreateJob).not.toHaveBeenCalled(); }); it('uses StrippedTextReply when available', async () => { diff --git a/apps/web/src/app/api/webhooks/postmark/route.ts b/apps/web/src/app/api/webhooks/postmark/route.ts index b076d06e..8259cebd 100644 --- a/apps/web/src/app/api/webhooks/postmark/route.ts +++ b/apps/web/src/app/api/webhooks/postmark/route.ts @@ -90,8 +90,9 @@ export async function POST(request: Request) { }); } - // Enqueue AI response for the reply - await createJob(JobType.AI_RESPONSE, { ticketId: existingTicket.id, source: 'web' }); + // No AI response on a reply — Outpost answers the opening email + // once and a human handles the rest of the thread. The + // AI_RESPONSE handler enforces the same invariant server-side. return NextResponse.json({ status: 'message_appended', ticketId: existingTicket.displayId }); } diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index c29d02b1..9fd4f959 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -857,6 +857,144 @@ describe('handleAiResponse', () => { }), ); }); + + // ── One response per ticket ─────────────────────────────────────────── + // + // The invariant: Outpost answers the message that opens a ticket and never + // posts in that thread again, whoever speaks next. The enqueue sites no + // longer queue on replies, but this guard is what makes the rule hold — it + // reads the ticket's own history, so a caller added later cannot route + // around it. + // + // The pipeline is mocked at the class seam here (not driven through LLMock) + // on purpose: the assertion these tests exist to make is that NO model call + // happens at all, and `mockGenerateSupportResponse` not being called is the + // direct expression of that. + describe('one response per ticket', () => { + /** A ticket that already carries the AI's single answer. */ + const answeredTicket = { + ...sampleTicket, + messages: [ + { + id: 'msg-1', + type: 'USER', + content: 'How do I use CopilotKit with Next.js?', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:00Z'), + }, + { + id: 'msg-2', + type: 'BOT', + content: 'Here is how to use CopilotKit with Next.js...', + isAiGenerated: true, + createdAt: new Date('2026-04-23T10:00:20Z'), + }, + ], + }; + + it('skips generation when the ticket already has an AI response', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data).toMatchObject({ skipped: true, reason: 'already_answered' }); + expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + }); + + it('does not post anything to the platform for an already-answered ticket', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockPostResponse).not.toHaveBeenCalled(); + expect(mockPrismaMessage.create).not.toHaveBeenCalled(); + expect(mockPrismaTicket.update).not.toHaveBeenCalled(); + }); + + it('reports success so the job is not retried forever', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + // A failure verdict would put an unchangeable decision through the + // retry ladder. Asserting `skipped` alongside it matters: without it + // this test also passes on the ordinary answer path, so it would + // stop proving anything if the guard were removed. + expect(result.success).toBe(true); + expect(result.error).toBeUndefined(); + expect(result.data).toMatchObject({ skipped: true }); + }); + + it('skips even when a human replied after the AI response', async () => { + // The exact case that prompted this: a maintainer posted the real + // solution, and the bot answered again 14 seconds later. + mockPrismaTicket.findUnique.mockResolvedValue({ + ...answeredTicket, + messages: [ + ...answeredTicket.messages, + { + id: 'msg-3', + type: 'USER', + content: 'Here is the actual fix, from a maintainer.', + isAiGenerated: false, + createdAt: new Date('2026-05-06T20:06:09Z'), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.data).toMatchObject({ skipped: true }); + expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + }); + + it('still answers a ticket whose only messages are from users', async () => { + // Guard must not swallow the first, legitimate response. + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.data).not.toMatchObject({ skipped: true }); + expect(mockGenerateSupportResponse).toHaveBeenCalled(); + expect(mockPostResponse).toHaveBeenCalled(); + }); + + it('does not treat a SYSTEM shadow-mode log as the ticket answer', async () => { + // Shadow mode writes SYSTEM + isAiGenerated rows alongside the BOT + // row. Only the BOT row means "the reporter has been answered", so a + // ticket carrying just a SYSTEM row must still be answerable. + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-shadow', + type: 'SYSTEM', + content: 'shadow log', + isAiGenerated: true, + createdAt: new Date('2026-04-23T10:00:10Z'), + }, + ], + }); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockGenerateSupportResponse).toHaveBeenCalled(); + }); + }); }); /** diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index e7d44548..031190e6 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -75,6 +75,39 @@ export async function handleAiResponse( await context.reportProgress(20); + // 1b. ONE RESPONSE PER TICKET — hard invariant, enforced here. + // + // Outpost answers exactly one message per ticket: the one that opened it. + // Every later message in that thread gets no AI reply, no matter who sent + // it — the original reporter, a third party, or a team member. The agent is + // a first line of defence and a human owns the thread from the moment the + // first response lands. + // + // The gate lives in the handler rather than at the enqueue sites on + // purpose. Five separate code paths could enqueue AI_RESPONSE (Discord, + // Slack, Teams, the GitHub comment webhook, the Postmark reply webhook) and + // each one previously decided for itself whether a reply warranted an + // answer. Those enqueues are gone, but a single new caller added later + // would silently reintroduce the follow-up spam this closes. Checking the + // ticket's own history instead makes the invariant unroutable-around. + // + // Success, not failure: the job did what it should — nothing. Returning an + // error would put it through the retry ladder for a decision that will + // never change. + const priorAiResponse = ticket.messages.find( + (m: { type: string; isAiGenerated: boolean }) => m.type === 'BOT' && m.isAiGenerated, + ); + if (priorAiResponse) { + console.log( + `[AI Response] Ticket ${ticketId} already answered — skipping. ` + + `Outpost posts one response per ticket; a human owns this thread now.`, + ); + return { + success: true, + data: { ticketId, skipped: true, reason: 'already_answered' }, + }; + } + // 2. Build conversation history from DB messages const conversationHistory = ticket.messages .filter((m: { type: string }) => m.type !== 'SYSTEM') diff --git a/packages/outpost/shared/src/__tests__/inbound-handler.test.ts b/packages/outpost/shared/src/__tests__/inbound-handler.test.ts index 9fb3aee8..a8a44fc7 100644 --- a/packages/outpost/shared/src/__tests__/inbound-handler.test.ts +++ b/packages/outpost/shared/src/__tests__/inbound-handler.test.ts @@ -207,14 +207,13 @@ describe('InboundHandler (GitHub-focused)', () => { }); }); - it('enqueues AI_RESPONSE for non-team-member follow-ups', async () => { + // One response per ticket: the issue body gets an answer, comments do not. + it('never enqueues AI_RESPONSE for follow-ups', async () => { const message = makeFollowUpMessage(); - await handler.handle(message); + const result = await handler.handle(message); - expect(createJob).toHaveBeenCalledWith('AI_RESPONSE', expect.objectContaining({ - ticketId: 'ticket-existing', - source: 'github', - })); + expect(result.aiJobEnqueued).toBe(false); + expect(createJob).not.toHaveBeenCalled(); }); it('creates new ticket if no existing ticket found for reply thread', async () => { diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index bd5c0698..7016d482 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -302,16 +302,38 @@ describe('InboundHandler', () => { expect(msgData.content).toBe('Follow up question'); }); - it('enqueues AI_RESPONSE for non-team-member reply', async () => { + // ONE RESPONSE PER TICKET. Outpost answers the message that opens a + // ticket and stays out of the thread after that — no follow-up reply to + // anyone. Before this, every non-team reply enqueued an AI_RESPONSE, so + // the bot butted into follow-up questions between community members and + // summarised a human's answer back at them. + it('never enqueues AI_RESPONSE for a reply from a non-team member', async () => { const msg = makeInboundMessage({ isThreadStart: false }); const result = await handler.handle(msg); - expect(result.aiJobEnqueued).toBe(true); - expect(createJob).toHaveBeenCalledWith('AI_RESPONSE', { - ticketId: 'ticket-existing', - threadId: 'thread-123', - source: 'discord', + expect(result.aiJobEnqueued).toBe(false); + expect(createJob).not.toHaveBeenCalled(); + }); + + it('never enqueues AI_RESPONSE for a reply from an unrelated third party', async () => { + const msg = makeInboundMessage({ + isThreadStart: false, + platformUserId: 'someone-else-999', + platformUsername: 'bystander', + content: 'did you ever get this working?', }); + const result = await handler.handle(msg); + + expect(result.aiJobEnqueued).toBe(false); + expect(createJob).not.toHaveBeenCalled(); + }); + + it('still appends the reply as a Message even though no AI job is queued', async () => { + const msg = makeInboundMessage({ isThreadStart: false, content: 'any update?' }); + await handler.handle(msg); + + expect(prisma.message.create).toHaveBeenCalledTimes(1); + expect(createJob).not.toHaveBeenCalled(); }); it('skips AI_RESPONSE for team member reply', async () => { @@ -469,6 +491,10 @@ describe('InboundHandler', () => { // ── Team member detection ──────────────────────────────────────── + // These exercise isTeamMember through the NEW-ticket path, because that is + // the only path where the flag still varies. Replies never enqueue an + // AI_RESPONSE regardless of sender, so asserting aiJobEnqueued on a reply + // would pass no matter what isTeamMember returned. describe('team member detection', () => { it('identifies team member by User.externalId + TeamMember.email lookup', async () => { (prisma.user.findFirst as ReturnType).mockResolvedValue({ @@ -479,19 +505,7 @@ describe('InboundHandler', () => { id: 'member-1', }); - const msg = makeInboundMessage({ isThreadStart: false }); - - // Set up existing ticket for reply - (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ - id: 'ticket-1', - displayId: 'TKT-TEST1234', - status: 'OPEN', - sourceId: 'thread-123', - channel: 'channel-1', - source: 'DISCORD', - }); - - const result = await handler.handle(msg); + const result = await handler.handle(makeInboundMessage()); expect(result.aiJobEnqueued).toBe(false); // Verify User lookup used correct source @@ -509,34 +523,14 @@ describe('InboundHandler', () => { email: null, }); - (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ - id: 'ticket-1', - displayId: 'TKT-TEST1234', - status: 'OPEN', - sourceId: 'thread-123', - channel: 'channel-1', - source: 'DISCORD', - }); - - const msg = makeInboundMessage({ isThreadStart: false }); - const result = await handler.handle(msg); + const result = await handler.handle(makeInboundMessage()); expect(result.aiJobEnqueued).toBe(true); }); it('returns false when user not found in database', async () => { (prisma.user.findFirst as ReturnType).mockResolvedValue(null); - (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ - id: 'ticket-1', - displayId: 'TKT-TEST1234', - status: 'OPEN', - sourceId: 'thread-123', - channel: 'channel-1', - source: 'DISCORD', - }); - - const msg = makeInboundMessage({ isThreadStart: false }); - const result = await handler.handle(msg); + const result = await handler.handle(makeInboundMessage()); expect(result.aiJobEnqueued).toBe(true); }); @@ -547,17 +541,7 @@ describe('InboundHandler', () => { }); (prisma.teamMember.findUnique as ReturnType).mockResolvedValue(null); - (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ - id: 'ticket-1', - displayId: 'TKT-TEST1234', - status: 'OPEN', - sourceId: 'thread-123', - channel: 'channel-1', - source: 'DISCORD', - }); - - const msg = makeInboundMessage({ isThreadStart: false }); - const result = await handler.handle(msg); + const result = await handler.handle(makeInboundMessage()); expect(result.aiJobEnqueued).toBe(true); }); }); diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 8d51dc6d..27ebabf9 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -4,8 +4,9 @@ * * Handles: * 1. New tickets (isThreadStart=true): create Ticket + first Message + enqueue AI_RESPONSE - * 2. Replies (isThreadStart=false): find existing ticket, create Message, reopen if needed, - * enqueue AI_RESPONSE unless sender is a team member + * 2. Replies (isThreadStart=false): find existing ticket, create Message, reopen if needed. + * Never enqueues AI_RESPONSE — Outpost answers once per ticket, on the opening + * message only, and a human owns the thread after that. * 3. Team member detection via ExternalIdentity -> TeamMember lookup * 4. Sequential display ID generation (TKT-XXXXXXXX) */ @@ -238,11 +239,21 @@ export class InboundHandler { }, }); - // Check if sender is a team member + // NO AI RESPONSE ON REPLIES — deliberate, not an omission. + // + // Outpost answers the message that opens a ticket and nothing after it. + // Replies only move ticket state; the thread belongs to a human from + // the first response onward. This used to enqueue an AI_RESPONSE for + // every non-team sender, which meant the bot chimed in on follow-up + // questions between community members and even summarised a human's + // answer back at them. + // + // The invariant is also enforced in the AI_RESPONSE handler + // (packages/outpost/queue/src/handlers/ai-response.ts) against the + // ticket's own message history. Not enqueuing here is the cheap arm — + // it avoids paying for a job that would be dropped on arrival. const isTeam = await this.isTeamMember(message.platformUserId, message.source); - let aiJobEnqueued = false; - if (isTeam) { // Team member replied: if ticket was WAITING_ON_TEAM, move to WAITING_ON_CUSTOMER if (ticket.status === 'WAITING_ON_TEAM') { @@ -251,29 +262,23 @@ export class InboundHandler { data: { status: 'WAITING_ON_CUSTOMER' }, }); } - } else { - // Customer/external user replied: enqueue AI response - await this.createJob(this.aiResponseJobType, { - ticketId: ticket.id, - threadId: message.threadId, - source: toPlatformTarget(message.source), + } else if ( + ticket.status === 'WAITING_ON_CUSTOMER' || + ticket.status === 'RESOLVED' || + ticket.status === 'CLOSED' + ) { + // Customer/external reply reopens a dormant ticket so a human sees it. + await this.prisma.ticket.update({ + where: { id: ticket.id }, + data: { status: 'OPEN' }, }); - aiJobEnqueued = true; - - // Reopen the ticket if it was waiting on customer, resolved, or closed - if (ticket.status === 'WAITING_ON_CUSTOMER' || ticket.status === 'RESOLVED' || ticket.status === 'CLOSED') { - await this.prisma.ticket.update({ - where: { id: ticket.id }, - data: { status: 'OPEN' }, - }); - } } return { ticketId: ticket.id, displayId: ticket.displayId, isNewTicket: false, - aiJobEnqueued, + aiJobEnqueued: false, messageId: msg.id, }; } From ae314deaa4459a53034fa891a7a8e053a1e75995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:23:50 -0400 Subject: [PATCH 57/83] fix(teams-bot): use real em dash in two card doc comments The escalation and ticket-created card doc comments carried the literal six-character sequence \u2014 where an em dash was meant; in a comment that renders as raw escape text. response-card.ts already used the real character, so the diff was internally inconsistent. Comment-only; no behavior change. The escape sequences in the card bodies' text: string values are the pre-existing convention and are left as-is. (cherry picked from commit ac6540eaeec0cb16f854fae58bf4ad90b70b24ae) --- apps/teams-bot/src/cards/escalation-card.ts | 2 +- apps/teams-bot/src/cards/ticket-created-card.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/teams-bot/src/cards/escalation-card.ts b/apps/teams-bot/src/cards/escalation-card.ts index 7499fe30..53fe9c47 100644 --- a/apps/teams-bot/src/cards/escalation-card.ts +++ b/apps/teams-bot/src/cards/escalation-card.ts @@ -5,7 +5,7 @@ export interface EscalationCardOptions { /** * Build an Adaptive Card for escalation notifications. * - * Carries no ticket displayId \u2014 this card is shown to the reporter, and the + * Carries no ticket displayId — this card is shown to the reporter, and the * identifier is internal to the dashboard and team slash commands. */ export function buildEscalationCard(options: EscalationCardOptions): Record { diff --git a/apps/teams-bot/src/cards/ticket-created-card.ts b/apps/teams-bot/src/cards/ticket-created-card.ts index fa1bc66c..b7d19a3a 100644 --- a/apps/teams-bot/src/cards/ticket-created-card.ts +++ b/apps/teams-bot/src/cards/ticket-created-card.ts @@ -5,7 +5,7 @@ export interface TicketCreatedCardOptions { /** * Build an Adaptive Card acknowledging ticket creation. * - * Deliberately carries no ticket displayId. That identifier is internal \u2014 it + * Deliberately carries no ticket displayId. That identifier is internal — it * belongs in the dashboard and team slash commands, not in reporter-facing copy. */ export function buildTicketCreatedCard(options: TicketCreatedCardOptions): Record { From 4dfde3c3830484f3f9e50f74f4e485e9cf5025ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:25:54 -0400 Subject: [PATCH 58/83] fix(teams): drop dead ticketDisplayId from response card payloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doc comment added in e415e31 claimed the internal ticketDisplayId was "routed through the Action.Submit `data` payloads so button clicks resolve back to the ticket". That was false. Both handlers in apps/teams-bot/src/handlers/card-actions.ts take the payload as `_data` (unused) and resolve the ticket via findTicketByConversationId(context.activity.conversation.id). Nothing ever read data.ticketDisplayId, so the identifier was dead payload that still shipped to the reporter's Teams client inside the card JSON. Removed it from ResponseCardOptions, from both Action.Submit `data` payloads, from PostResponseOptions in lib/teams-poster.ts, and from CardActionData (the receiving half of the same dead field — it declared a required string that never arrives). Doc comments now state what is true: the card carries no identifier anywhere, the handlers resolve by conversation id, and this module is currently unreachable in production — nothing imports teams-poster.ts and the worker posts PlatformTeamsAdapter.buildResponseCard from packages/outpost/shared/src/platforms/teams.ts instead. Module kept, just made correct. Call-site enumeration (grep 'ticketDisplayId|buildResponseCard|postAiResponse|ResponseCardOptions|PostResponseOptions' across the repo): - ResponseCardOptions.ticketDisplayId (removed) — only producer was lib/teams-poster.ts, updated in this commit. No other reference. Holds. - buildResponseCard (apps/teams-bot/src/cards/response-card.ts) — two references: lib/teams-poster.ts (updated) and src/__tests__/cards.test.ts (updated). Both now pass only {responseText, confidence}. Holds; tsc --noEmit is clean, which is the proof there is no third caller. - PostResponseOptions.ticketDisplayId (removed) — NEGATIVE FINDING: nothing imports lib/teams-poster.ts at all (no src file, no test). postAiResponse here has zero call sites, so removing the field breaks no caller. - CardActionData.ticketDisplayId (removed) — only read site was the destructure-free `_data` params of handleIssueSolved / handleNeedMoreHelp, both of which ignore it. NEGATIVE FINDING: no surviving card emits the field either — buildEscalationCard and buildTicketCreatedCard define no `actions`, and PlatformTeamsAdapter.buildResponseCard (packages/outpost/shared/src/platforms/teams.ts:269) returns a card with no `actions` array at all, so the production path never sends button payloads. No backward-compat payload is lost. - Same-named symbols in sibling apps are unrelated and untouched: apps/slack-bot/src/lib/slack-poster.ts still uses ticketDisplayId for Slack block_id / action `value` (Slack routes by action value there, a genuinely live use), and apps/github-app/src/lib/github-poster.ts has its own PostResponseOptions with no such field. Neither imports the Teams card module. Assumptions unaffected. - Test fixture apps/teams-bot/src/__tests__/card-actions.test.ts makeCardContext no longer stuffs ticketDisplayId into activity.value, so the suite now proves the handlers resolve the ticket without it. Tests: the old "still routes the displayId through action data so clicks resolve" case encoded the false claim and is replaced by "carries no ticket identifier anywhere — not in text, not in action data", which asserts Object.keys(action.data) === ['action'] for both buttons plus a JSON.stringify catch-all against /[Dd]isplayId|TKT-/. Red-green: re-added ticketDisplayId to the issue_solved payload → new test failed on the action-data key assertion (RED); restored → 55/55 pass (GREEN). `npx vitest run --reporter=dot` and `npx tsc --noEmit` both clean in apps/teams-bot. (cherry picked from commit bf5f70e609ac31c62f26a78b401b3e499e69cdb8) --- .../src/__tests__/card-actions.test.ts | 3 ++- apps/teams-bot/src/__tests__/cards.test.ts | 26 +++++++------------ apps/teams-bot/src/cards/response-card.ts | 24 ++++++++++------- apps/teams-bot/src/handlers/card-actions.ts | 6 ++++- apps/teams-bot/src/lib/teams-poster.ts | 11 +++++--- 5 files changed, 40 insertions(+), 30 deletions(-) diff --git a/apps/teams-bot/src/__tests__/card-actions.test.ts b/apps/teams-bot/src/__tests__/card-actions.test.ts index 75580aec..463f64d3 100644 --- a/apps/teams-bot/src/__tests__/card-actions.test.ts +++ b/apps/teams-bot/src/__tests__/card-actions.test.ts @@ -30,9 +30,10 @@ function makeCardContext(action: string, overrides: Record = {} return { activity: { type: 'invoke', + // The cards send only `action` — no ticket identifier. The handlers + // resolve the ticket from `conversation.id`. value: { action, - ticketDisplayId: 'TKT-AB12', }, from: { id: 'user-456', diff --git a/apps/teams-bot/src/__tests__/cards.test.ts b/apps/teams-bot/src/__tests__/cards.test.ts index 1a863d87..e653736d 100644 --- a/apps/teams-bot/src/__tests__/cards.test.ts +++ b/apps/teams-bot/src/__tests__/cards.test.ts @@ -15,7 +15,6 @@ function allCardText(card: Record): string { describe('buildResponseCard', () => { it('builds a card with action buttons', () => { const card = buildResponseCard({ - ticketDisplayId: 'TKT-AB12', responseText: 'Here is the answer.', confidence: 0.9, }); @@ -35,31 +34,27 @@ describe('buildResponseCard', () => { expect(actions[1].data.action).toBe('need_more_help'); }); - it('never renders the internal ticket displayId into card text', () => { + it('carries no ticket identifier anywhere — not in text, not in action data', () => { const card = buildResponseCard({ - ticketDisplayId: 'TKT-AB12', responseText: 'Here is the answer.', confidence: 0.9, }); - expect(allCardText(card)).not.toContain('TKT-AB12'); - }); + expect(allCardText(card)).not.toMatch(/TKT-/); - it('still routes the displayId through action data so clicks resolve', () => { - const card = buildResponseCard({ - ticketDisplayId: 'TKT-AB12', - responseText: 'Here is the answer.', - confidence: 0.9, - }); + // `data` ships to the reporter's client too, so the action payloads must + // carry nothing but the action name. + const actions = card.actions as Array<{ data: Record }>; + for (const action of actions) { + expect(Object.keys(action.data)).toEqual(['action']); + } - const actions = card.actions as Array<{ data: { ticketDisplayId: string } }>; - expect(actions[0].data.ticketDisplayId).toBe('TKT-AB12'); - expect(actions[1].data.ticketDisplayId).toBe('TKT-AB12'); + // Catch-all: no identifier-shaped field anywhere in the serialized card. + expect(JSON.stringify(card)).not.toMatch(/[Dd]isplayId|TKT-/); }); it('includes a low-confidence disclaimer when confidence is below threshold', () => { const card = buildResponseCard({ - ticketDisplayId: 'TKT-AB12', responseText: 'Not sure about this.', confidence: 0.5, }); @@ -71,7 +66,6 @@ describe('buildResponseCard', () => { it('does not include disclaimer when confidence is high', () => { const card = buildResponseCard({ - ticketDisplayId: 'TKT-AB12', responseText: 'Confident answer.', confidence: 0.85, }); diff --git a/apps/teams-bot/src/cards/response-card.ts b/apps/teams-bot/src/cards/response-card.ts index 9e49c8d6..b09c6bc6 100644 --- a/apps/teams-bot/src/cards/response-card.ts +++ b/apps/teams-bot/src/cards/response-card.ts @@ -1,10 +1,4 @@ export interface ResponseCardOptions { - /** - * Internal ticket identifier. Routed through the Action.Submit `data` - * payloads so button clicks resolve back to the ticket — never rendered - * into card text, which the reporter reads. - */ - ticketDisplayId: string; responseText: string; confidence: number; } @@ -14,9 +8,23 @@ const LOW_CONFIDENCE_THRESHOLD = 0.7; /** * Build an Adaptive Card JSON for an AI response. * Includes action buttons and an optional low-confidence disclaimer. + * + * Carries no ticket identifier anywhere — not in card text and not in the + * `Action.Submit` `data` payloads. The whole card, `data` included, ships to + * the reporter's Teams client, and the identifier is internal to the dashboard + * and team slash commands. Nothing needs it here either: the button handlers in + * `../handlers/card-actions.ts` resolve the ticket from + * `context.activity.conversation.id`, so the only field the payload has to + * carry is `action`. + * + * Reachability: this module is currently unreachable in production. Its only + * importer is `../lib/teams-poster.ts`, which nothing imports; the worker posts + * AI responses through `PlatformTeamsAdapter.buildResponseCard` in + * `packages/outpost/shared/src/platforms/teams.ts` instead. Kept and kept + * correct rather than deleted. */ export function buildResponseCard(options: ResponseCardOptions): Record { - const { ticketDisplayId, responseText, confidence } = options; + const { responseText, confidence } = options; const body: Record[] = [ { @@ -53,7 +61,6 @@ export function buildResponseCard(options: ResponseCardOptions): Record { - const { context, ticketDisplayId, responseText, confidence } = options; + const { context, responseText, confidence } = options; const card = buildResponseCard({ - ticketDisplayId, responseText, confidence, }); From ebc503a013b14468815d49218e9b2b4c7491b104 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:27:49 -0400 Subject: [PATCH 59/83] test(teams-bot): make the displayId leak guards able to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guards added with the leak fix could not detect a regression: - allCardText only read top-level body[].text, skipping nested containers and every actions[].title. - The ticket-created and escalation guards were tautological — neither builder accepts a displayId, so no input could have carried a leak. Replace the helper with visibleCardStrings, a recursive walk over the whole card that collects every string except Action.Submit data payloads (a denylist, so an unknown rendered field still trips the guard), and add a meta-test pinning that the walker reaches nested containers and action titles. Feed each guard an input that WOULD leak if the builder passed it through: the response card already receives the displayId (now checked in both the high- and low-confidence body shapes), and the ack / escalation guards hand the builders a ticketDisplayId property they currently ignore. Also pin that title/reason are echoed verbatim, so the caller-owned boundary is documented rather than assumed. Tests only — no production behavior change. (cherry picked from commit 16b38d2639be56cb6dfe599424efb414ec754e94) --- apps/teams-bot/src/__tests__/cards.test.ts | 144 ++++++++++++++++++--- 1 file changed, 125 insertions(+), 19 deletions(-) diff --git a/apps/teams-bot/src/__tests__/cards.test.ts b/apps/teams-bot/src/__tests__/cards.test.ts index e653736d..ad7a767e 100644 --- a/apps/teams-bot/src/__tests__/cards.test.ts +++ b/apps/teams-bot/src/__tests__/cards.test.ts @@ -1,17 +1,75 @@ import { describe, it, expect } from 'vitest'; import { buildResponseCard } from '../cards/response-card.js'; -import { buildTicketCreatedCard } from '../cards/ticket-created-card.js'; -import { buildEscalationCard } from '../cards/escalation-card.js'; +import { + buildTicketCreatedCard, + type TicketCreatedCardOptions, +} from '../cards/ticket-created-card.js'; +import { buildEscalationCard, type EscalationCardOptions } from '../cards/escalation-card.js'; /** - * Every text block in a card body, flattened. Used by the leak assertions - * below so a displayId can't reappear in a block index nobody checks. + * Key whose value is an Action.Submit payload: sent back to the bot on click, + * never displayed. Everything else in a card is potentially rendered, so the + * leak guards below look at all of it. */ -function allCardText(card: Record): string { - const body = card.body as Array<{ text?: string }>; - return body.map((b) => b.text ?? '').join('\n'); +const SUBMIT_PAYLOAD_KEY = 'data'; + +/** + * Every string a Teams client could put on screen for this card, at any nesting + * depth: `body[].text`, text inside nested containers/columns/sets, and + * `actions[].title` — plus any other string field a future card shape adds. + * + * Deliberately a denylist (skip `data`) rather than an allowlist of known + * rendered keys, so a displayId reintroduced into a field this test has never + * heard of still trips the guard. + */ +function visibleCardStrings(value: unknown): string[] { + if (typeof value === 'string') return [value]; + if (Array.isArray(value)) return value.flatMap(visibleCardStrings); + if (value !== null && typeof value === 'object') { + return Object.entries(value as Record) + .filter(([key]) => key !== SUBMIT_PAYLOAD_KEY) + .flatMap(([, nested]) => visibleCardStrings(nested)); + } + return []; } +/** The visible strings joined, for substring/regex assertions. */ +function visibleCardText(card: Record): string { + return visibleCardStrings(card).join('\n'); +} + +describe('visibleCardStrings (the leak guards depend on this)', () => { + it('finds text nested inside containers and action titles, and skips submit payloads', () => { + const card = { + type: 'AdaptiveCard', + body: [ + { type: 'TextBlock', text: 'top-level' }, + { + type: 'Container', + items: [ + { type: 'TextBlock', text: 'nested-once' }, + { type: 'ColumnSet', columns: [{ items: [{ text: 'nested-twice' }] }] }, + ], + }, + ], + actions: [ + { + type: 'Action.Submit', + title: 'action-title', + data: { ticketDisplayId: 'TKT-INPAYLOAD' }, + }, + ], + }; + + const found = visibleCardStrings(card); + expect(found).toContain('top-level'); + expect(found).toContain('nested-once'); + expect(found).toContain('nested-twice'); + expect(found).toContain('action-title'); + expect(found).not.toContain('TKT-INPAYLOAD'); + }); +}); + describe('buildResponseCard', () => { it('builds a card with action buttons', () => { const card = buildResponseCard({ @@ -34,22 +92,38 @@ describe('buildResponseCard', () => { expect(actions[1].data.action).toBe('need_more_help'); }); - it('carries no ticket identifier anywhere — not in text, not in action data', () => { + // The card no longer accepts a displayId at all, so the leak-carrying input + // has to arrive through a field that still exists. `responseText` is the + // pipeline's own output, which is exactly where a stray identifier could + // come from in practice. + it.each([ + ['high confidence', 0.9], + ['low confidence (extra disclaimer block)', 0.5], + ])('renders no identifier-shaped string anywhere visible — %s', (_label, confidence) => { const card = buildResponseCard({ responseText: 'Here is the answer.', - confidence: 0.9, + confidence, }); - expect(allCardText(card)).not.toMatch(/TKT-/); + expect(visibleCardText(card)).not.toMatch(/TKT-/); + }); + + it('carries nothing but the action name in its submit payloads', () => { + // `data` ships to the reporter's client too. It used to carry the ticket + // displayId on the claim that clicks needed it to resolve; card-actions.ts + // ignores `data` entirely and resolves by conversation id, so the field + // was dead payload. This pins it staying gone. + const card = buildResponseCard({ + responseText: 'Here is the answer.', + confidence: 0.9, + }); - // `data` ships to the reporter's client too, so the action payloads must - // carry nothing but the action name. const actions = card.actions as Array<{ data: Record }>; for (const action of actions) { expect(Object.keys(action.data)).toEqual(['action']); } - // Catch-all: no identifier-shaped field anywhere in the serialized card. + // Catch-all across the whole serialized card, submit payloads included. expect(JSON.stringify(card)).not.toMatch(/[Dd]isplayId|TKT-/); }); @@ -87,10 +161,28 @@ describe('buildTicketCreatedCard', () => { expect(body[2].text).toContain('AI assistant'); }); - it('never renders a ticket displayId — the option does not exist', () => { - const card = buildTicketCreatedCard({ title: 'Help with integration' }); + it('ignores a ticketDisplayId even when one is handed to it', () => { + // The regression this guards: someone re-adds a displayId to the options + // and renders it into reporter-facing copy. Passing the leak-carrying + // property today is a no-op; the day the builder reads it, this fails. + const card = buildTicketCreatedCard({ + title: 'Help with integration', + ticketDisplayId: 'TKT-LEAK01', + } as TicketCreatedCardOptions); - expect(allCardText(card)).not.toMatch(/TKT-/); + expect(visibleCardText(card)).not.toContain('TKT-LEAK01'); + expect(visibleCardText(card)).not.toMatch(/TKT-/); + }); + + it('echoes the caller-supplied title verbatim', () => { + // Documented boundary: `title` is the reporter's own message text + // (handlers/message.ts passes truncate(message.content)), so it is + // rendered as-is. Callers must never put an internal displayId here — + // this builder does not sanitize, and this test pins that contract. + const card = buildTicketCreatedCard({ title: 'my ref is TKT-USERTYPED' }); + + const body = card.body as Array<{ text: string }>; + expect(body[1].text).toBe('my ref is TKT-USERTYPED'); }); }); @@ -106,9 +198,23 @@ describe('buildEscalationCard', () => { expect(body[2].text).toContain('team member'); }); - it('never renders a ticket displayId — the option does not exist', () => { - const card = buildEscalationCard({ reason: 'User needs more help.' }); + it('ignores a ticketDisplayId even when one is handed to it', () => { + const card = buildEscalationCard({ + reason: 'User needs more help.', + ticketDisplayId: 'TKT-LEAK02', + } as EscalationCardOptions); + + expect(visibleCardText(card)).not.toContain('TKT-LEAK02'); + expect(visibleCardText(card)).not.toMatch(/TKT-/); + }); + + it('echoes the caller-supplied reason verbatim', () => { + // Same boundary as the ack card: `reason` is rendered as-is (the only + // call site passes a static literal), so callers own keeping displayIds + // out of it. + const card = buildEscalationCard({ reason: 'escalated from TKT-CALLERTEXT' }); - expect(allCardText(card)).not.toMatch(/TKT-/); + const body = card.body as Array<{ text: string }>; + expect(body[1].text).toBe('escalated from TKT-CALLERTEXT'); }); }); From 747cce08c41da513de0c34adee75ee57310f5dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:33:04 -0400 Subject: [PATCH 60/83] fix(inbound): one shared reopen-status set for all reply paths The three inbound reply paths each carried their own literal list of the ticket statuses a customer reply reopens, and they had drifted: shared InboundHandler WAITING_ON_CUSTOMER, RESOLVED, CLOSED (reference) github-app issue-comment WAITING_ON_CUSTOMER, RESOLVED (no CLOSED) postmark webhook RESOLVED, CLOSED (no WAITING_ON_CUSTOMER) That is now load-bearing: replies no longer enqueue an AI_RESPONSE, so the reopen is the ONLY signal a customer follow-up sends to a human. A comment on a CLOSED GitHub ticket reached nobody, and an email reply to a ticket waiting on the customer stayed out of the queue. Fix: one exported predicate in @copilotkit/outpost/shared - REOPEN_ON_CUSTOMER_REPLY_STATUSES + reopensOnCustomerReply(status) - and all three paths call it. No path keeps a literal status list. Tests (red-green verified): - shared: CLOSED reopen case (was uncovered) + a reopensOnCustomerReply unit block, including an exhaustiveness check over TicketStatus so a new status cannot be added without deciding whether a reply reopens it. - github-app: it.each over the three dormant statuses (CLOSED was uncovered) plus a negative it.each over OPEN/IN_PROGRESS/WAITING_ON_TEAM. - apps/web postmark: same pair (WAITING_ON_CUSTOMER was uncovered). - Both app tests now spread importActual over @copilotkit/outpost/shared instead of stubbing it wholesale, so they exercise the real predicate. - RED confirmed three ways: reverting the github condition fails 1 github test; reverting the postmark condition fails 1 web test; dropping CLOSED from the shared constant fails 3 shared + 1 github + 1 web test. GREEN restored after each. Verification: packages/outpost 969 tests / 60 files pass, typecheck clean; apps/github-app 43 tests pass, tsc --noEmit clean; apps/web 541 tests / 47 files pass, tsc --noEmit clean; eslint clean on the four source files. Call-Site Enumeration --------------------- Added: REOPEN_ON_CUSTOMER_REPLY_STATUSES, reopensOnCustomerReply (packages/outpost/shared/src/constants.ts). Already public: constants.ts is re-exported wholesale by shared/src/index.ts ("export * from './constants.js'"), so no new line was needed in the entry point; verified by apps/web and apps/github-app importing from '@copilotkit/outpost/shared' and typechecking. References to the two new symbols (every one): - shared/src/platforms/inbound.ts:16,266 - handleReply. Holds: the value is the exact set this file used to inline, so the reference-set behaviour is unchanged; its existing WAITING_ON_CUSTOMER/RESOLVED tests plus a new CLOSED test pass. - shared/src/__tests__/platforms-inbound.test.ts:6,621-653 - new unit block. Holds: asserts the set contents, the negative statuses, null/undefined safety, and TicketStatus exhaustiveness. - apps/github-app/src/webhooks/issue-comment.ts:4,89 - non-team commenter branch. Holds: ticket.status comes off the Prisma row as a string, which is exactly the predicate's parameter type; the team-member branch above is untouched (WAITING_ON_TEAM -> WAITING_ON_CUSTOMER stays its own rule). - apps/web/src/app/api/webhooks/postmark/route.ts:13,89 - existing-ticket reply branch. Holds: same string status; the "updatedAt: new Date()" in the update payload is preserved. This path has no team-member detection, so every inbound email on a ticket is a customer reply by construction - applying the customer predicate is correct here. - apps/web/src/__tests__/postmark-webhook.test.ts:28 - comment only. Removed: three inline status literals. No exported symbol was removed, so no external caller could depend on them. Confirmed by grep that no literal reopen set remains - "=== 'RESOLVED'" / "=== 'CLOSED'" now hit only two unrelated sites. Negative findings (checked, deliberately NOT converted): - packages/outpost/shared/src/sla/checker.ts:67 - "CLOSED || RESOLVED" stops the SLA clock. Different question (is the ticket finished), and it must not include WAITING_ON_CUSTOMER; left alone. - apps/web/src/app/api/accounts/[id]/route.ts:43 - "CLOSED || RESOLVED" counts finished tickets for an account. Same reason; left alone. - apps/discord-bot, apps/slack-bot, apps/teams-bot - no literal reopen set; they all reply through the shared InboundHandler, so they pick the fix up for free. Their existing reopen tests still pass. - apps/linear-sync/src/webhooks/sync-handler.ts:19 - declares its own TicketStatus string union but never gates a reopen on it; no change. - apps/discord-bot/src/lib/shadow-mode.ts:72 and packages/outpost/db/src/seed.ts - write "status: 'OPEN'" on ticket CREATE, not a reopen; unaffected. (cherry picked from commit 4006b902c8e565883863b3a11e36be66b54e4266) --- .../src/__tests__/issue-comment.test.ts | 48 +++++++++++++- apps/github-app/src/webhooks/issue-comment.ts | 7 ++- .../src/__tests__/postmark-webhook.test.ts | 49 ++++++++++++++- .../src/app/api/webhooks/postmark/route.ts | 9 ++- .../src/__tests__/platforms-inbound.test.ts | 63 ++++++++++++++++++- packages/outpost/shared/src/constants.ts | 28 +++++++++ .../outpost/shared/src/platforms/inbound.ts | 7 +-- 7 files changed, 198 insertions(+), 13 deletions(-) diff --git a/apps/github-app/src/__tests__/issue-comment.test.ts b/apps/github-app/src/__tests__/issue-comment.test.ts index d0c83225..99bdb6e7 100644 --- a/apps/github-app/src/__tests__/issue-comment.test.ts +++ b/apps/github-app/src/__tests__/issue-comment.test.ts @@ -18,7 +18,11 @@ const mockParseInboundEvent = vi.fn().mockReturnValue({ const mockPostSystemMessage = vi.fn().mockResolvedValue(undefined); const mockPostResponse = vi.fn().mockResolvedValue(undefined); -vi.mock('@copilotkit/outpost/shared', () => ({ +// The reopen predicate is deliberately NOT stubbed — this webhook and the +// shared InboundHandler must agree on which statuses a customer reply reopens, +// so the test exercises the real shared implementation. +vi.mock('@copilotkit/outpost/shared', async (importActual) => ({ + ...(await importActual()), truncate: vi.fn((str: string, _len: number) => str), })); @@ -214,6 +218,48 @@ describe('handleIssueComment', () => { }); }); + // The three dormant statuses a customer comment must reopen. This path used + // to omit CLOSED, so a comment on a closed GitHub issue reached nobody: + // replies no longer trigger an AI response, which makes the reopen the only + // signal that gets a human's attention. + it.each(['WAITING_ON_CUSTOMER', 'RESOLVED', 'CLOSED'])( + 'reopens a %s ticket when a customer comments', + async (status) => { + vi.mocked(prisma.ticketExternalLink.findUnique).mockResolvedValue({ + id: 'link-1', + ticketId: 'ticket-1', + plugin: 'github', + externalId: 'CopilotKit/CopilotKit#42', + ticket: { ...TICKET, status }, + } as unknown as ReturnType extends Promise ? T : never); + + await handleIssueComment(makeEvent()); + + expect(prisma.ticket.update).toHaveBeenCalledWith({ + where: { id: 'ticket-1' }, + data: { status: 'OPEN' }, + }); + }, + ); + + it.each(['OPEN', 'IN_PROGRESS', 'WAITING_ON_TEAM'])( + 'does not touch the status of a %s ticket on a customer comment', + async (status) => { + vi.mocked(prisma.ticketExternalLink.findUnique).mockResolvedValue({ + id: 'link-1', + ticketId: 'ticket-1', + plugin: 'github', + externalId: 'CopilotKit/CopilotKit#42', + ticket: { ...TICKET, status }, + } as unknown as ReturnType extends Promise ? T : never); + + await handleIssueComment(makeEvent()); + + expect(prisma.message.create).toHaveBeenCalled(); + expect(prisma.ticket.update).not.toHaveBeenCalled(); + }, + ); + it('updates status when team member replies to WAITING_ON_TEAM ticket', async () => { const waitingTicket = { ...TICKET, status: 'WAITING_ON_TEAM' }; vi.mocked(prisma.ticketExternalLink.findUnique).mockResolvedValue({ diff --git a/apps/github-app/src/webhooks/issue-comment.ts b/apps/github-app/src/webhooks/issue-comment.ts index cf176dc2..79aad339 100644 --- a/apps/github-app/src/webhooks/issue-comment.ts +++ b/apps/github-app/src/webhooks/issue-comment.ts @@ -1,6 +1,7 @@ import type { EmitterWebhookEvent } from '@octokit/webhooks'; import { prisma } from '@copilotkit/outpost/db'; import { GitHubPlatformAdapter } from '@copilotkit/outpost/shared/platforms'; +import { reopensOnCustomerReply } from '@copilotkit/outpost/shared'; import { getOctokit } from '../lib/github-client.js'; import { findTicketBySourceId, isTeamMember } from '../lib/tickets.js'; import { isRepoAllowed } from '../lib/repo-allowlist.js'; @@ -82,8 +83,10 @@ export async function handleIssueComment( // replying to follow-ups on issues a human had already picked up. // The AI_RESPONSE handler enforces the same invariant server-side. - // Reopen ticket if it was waiting on customer or resolved - if (ticket.status === 'WAITING_ON_CUSTOMER' || ticket.status === 'RESOLVED') { + // Reopen a dormant ticket so a human sees the follow-up. The status + // set lives in @copilotkit/outpost/shared so this path, the shared + // InboundHandler, and the Postmark webhook cannot drift apart. + if (reopensOnCustomerReply(ticket.status)) { await prisma.ticket.update({ where: { id: ticket.id }, data: { status: 'OPEN' }, diff --git a/apps/web/src/__tests__/postmark-webhook.test.ts b/apps/web/src/__tests__/postmark-webhook.test.ts index 4eea1376..cf8b10e3 100644 --- a/apps/web/src/__tests__/postmark-webhook.test.ts +++ b/apps/web/src/__tests__/postmark-webhook.test.ts @@ -25,7 +25,11 @@ vi.mock('@copilotkit/outpost/db', () => ({ // ─── Mock generateTicketId ────────────────────────────────────────────────── -vi.mock('@copilotkit/outpost/shared', () => ({ +// reopensOnCustomerReply is deliberately NOT stubbed — this webhook and the +// shared InboundHandler must agree on which statuses a reply reopens, so the +// test exercises the real shared implementation. +vi.mock('@copilotkit/outpost/shared', async (importActual) => ({ + ...(await importActual()), generateTicketId: vi.fn().mockReturnValue('TKT-TESTID01'), })); @@ -249,6 +253,49 @@ describe('Postmark inbound webhook', () => { ); }); + // WAITING_ON_CUSTOMER used to be missing from this path's status list, so + // an email reply to a ticket that was waiting on the customer stayed out + // of the queue entirely — replies no longer trigger an AI response, so + // the reopen is the only signal that reaches a human. + it.each(['WAITING_ON_CUSTOMER', 'RESOLVED', 'CLOSED'])( + 're-opens a %s ticket on a new inbound reply', + async (status) => { + mockTicketFindUnique.mockResolvedValue({ + id: 'dormant-ticket', + displayId: 'TKT-DORMANT1', + status, + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-reopen' }); + mockTicketUpdate.mockResolvedValue({}); + + await POST(postmarkRequest(fullPayload({ MailboxHash: 'TKT-DORMANT1' }))); + + expect(mockTicketUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'dormant-ticket' }, + data: expect.objectContaining({ status: 'OPEN' }), + }), + ); + }, + ); + + it.each(['OPEN', 'IN_PROGRESS', 'WAITING_ON_TEAM'])( + 'leaves a %s ticket status untouched on a new inbound reply', + async (status) => { + mockTicketFindUnique.mockResolvedValue({ + id: 'live-ticket', + displayId: 'TKT-LIVE0001', + status, + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-append' }); + + await POST(postmarkRequest(fullPayload({ MailboxHash: 'TKT-LIVE0001' }))); + + expect(mockMessageCreate).toHaveBeenCalled(); + expect(mockTicketUpdate).not.toHaveBeenCalled(); + }, + ); + it('creates new ticket when MailboxHash ticket is not found', async () => { mockTicketFindUnique.mockResolvedValue(null); mockTicketCreate.mockResolvedValue({ diff --git a/apps/web/src/app/api/webhooks/postmark/route.ts b/apps/web/src/app/api/webhooks/postmark/route.ts index 8259cebd..08aa3343 100644 --- a/apps/web/src/app/api/webhooks/postmark/route.ts +++ b/apps/web/src/app/api/webhooks/postmark/route.ts @@ -10,7 +10,7 @@ import crypto from 'node:crypto'; import { NextResponse } from 'next/server'; import { prisma } from '@copilotkit/outpost/db'; -import { generateTicketId } from '@copilotkit/outpost/shared'; +import { generateTicketId, reopensOnCustomerReply } from '@copilotkit/outpost/shared'; import { createJob, JobType } from '@copilotkit/outpost/queue'; import { extractTicketId, extractEmail, extractName } from './utils'; import type { PostmarkInboundPayload } from './utils'; @@ -82,8 +82,11 @@ export async function POST(request: Request) { }, }); - // Re-open ticket if it was resolved or closed - if (existingTicket.status === 'RESOLVED' || existingTicket.status === 'CLOSED') { + // Re-open a dormant ticket so a human sees the reply. The status + // set lives in @copilotkit/outpost/shared so this path, the + // shared InboundHandler, and the GitHub App issue-comment + // webhook cannot drift apart. + if (reopensOnCustomerReply(existingTicket.status)) { await prisma.ticket.update({ where: { id: existingTicket.id }, data: { status: 'OPEN', updatedAt: new Date() }, diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index 7016d482..ef915e4e 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { InboundHandler } from '../platforms/inbound.js'; import type { PrismaLike, CreateJobFn } from '../platforms/inbound.js'; import type { InboundMessage } from '../platforms/types.js'; -import { TicketSource } from '../types.js'; +import { TicketSource, TicketStatus } from '../types.js'; +import { REOPEN_ON_CUSTOMER_REPLY_STATUSES, reopensOnCustomerReply } from '../constants.js'; // ── Mock Prisma ──────────────────────────────────────────────────────── @@ -423,6 +424,21 @@ describe('InboundHandler', () => { }); }); + it('reopens ticket from CLOSED when customer replies', async () => { + (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ + ...existingTicket, + status: 'CLOSED', + }); + + const msg = makeInboundMessage({ isThreadStart: false }); + await handler.handle(msg); + + expect(prisma.ticket.update).toHaveBeenCalledWith({ + where: { id: 'ticket-existing' }, + data: { status: 'OPEN' }, + }); + }); + it('does NOT reopen ticket if status is OPEN or IN_PROGRESS', async () => { for (const status of ['OPEN', 'IN_PROGRESS']) { const freshPrisma = createMockPrisma(); @@ -594,3 +610,48 @@ describe('InboundHandler', () => { }); }); }); + +// ── The shared reopen predicate ──────────────────────────────────────── +// +// All three inbound reply paths (this handler, the GitHub App issue-comment +// webhook, the Postmark inbound-email webhook) gate their reopen on this one +// predicate. They used to each carry their own literal status list and had +// drifted apart, which silently dropped customer follow-ups. + +describe('reopensOnCustomerReply', () => { + it('reopens exactly the three dormant statuses', () => { + expect(REOPEN_ON_CUSTOMER_REPLY_STATUSES).toEqual([ + 'WAITING_ON_CUSTOMER', + 'RESOLVED', + 'CLOSED', + ]); + for (const status of REOPEN_ON_CUSTOMER_REPLY_STATUSES) { + expect(reopensOnCustomerReply(status)).toBe(true); + } + }); + + it('leaves live statuses alone', () => { + for (const status of ['OPEN', 'IN_PROGRESS', 'WAITING_ON_TEAM']) { + expect(reopensOnCustomerReply(status)).toBe(false); + } + }); + + it('is safe on null/undefined/unknown status', () => { + expect(reopensOnCustomerReply(null)).toBe(false); + expect(reopensOnCustomerReply(undefined)).toBe(false); + expect(reopensOnCustomerReply('NOT_A_STATUS')).toBe(false); + }); + + it('covers every TicketStatus value exactly once, reopen or not', () => { + // Guard against a new TicketStatus being added without deciding + // whether a customer reply should reopen it. + const all = Object.values(TicketStatus) as string[]; + const reopening = all.filter((s) => reopensOnCustomerReply(s)); + expect(reopening.sort()).toEqual(['CLOSED', 'RESOLVED', 'WAITING_ON_CUSTOMER']); + expect(all.filter((s) => !reopensOnCustomerReply(s)).sort()).toEqual([ + 'IN_PROGRESS', + 'OPEN', + 'WAITING_ON_TEAM', + ]); + }); +}); diff --git a/packages/outpost/shared/src/constants.ts b/packages/outpost/shared/src/constants.ts index f31b2862..2eadd2a4 100644 --- a/packages/outpost/shared/src/constants.ts +++ b/packages/outpost/shared/src/constants.ts @@ -68,3 +68,31 @@ export const BACKOFF_MAX_MS = 300_000; /** Pagination defaults */ export const DEFAULT_PAGE_SIZE = 25; export const MAX_PAGE_SIZE = 100; + +/** + * Ticket statuses that a customer (non-team) reply reopens back to OPEN. + * + * Every inbound reply path must agree on this set. Since Outpost stopped + * answering replies (it responds to the opening message only), reopening the + * ticket is the ONLY signal a reply sends to a human — a path that omits a + * status here silently drops the customer's follow-up on the floor. + * + * Readers: the shared InboundHandler (`platforms/inbound.ts`), the GitHub App + * issue-comment webhook, and the Postmark inbound-email webhook. Do not inline + * the literal set anywhere; call `reopensOnCustomerReply` instead. + */ +export const REOPEN_ON_CUSTOMER_REPLY_STATUSES = [ + 'WAITING_ON_CUSTOMER', + 'RESOLVED', + 'CLOSED', +] as const satisfies readonly string[]; + +/** + * True when a customer reply to a ticket in `status` should reopen it. + * + * Takes a plain string (not TicketStatus) because callers read the status + * straight off a Prisma row, where it is typed as the DB enum / string. + */ +export function reopensOnCustomerReply(status: string | null | undefined): boolean { + return (REOPEN_ON_CUSTOMER_REPLY_STATUSES as readonly string[]).includes(status ?? ''); +} diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 27ebabf9..096be070 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -13,6 +13,7 @@ import type { InboundMessage, InboundResult, TicketRef } from './types.js'; import { generateTicketId, truncate } from '../utils.js'; +import { reopensOnCustomerReply } from '../constants.js'; import { TicketSource } from '../types.js'; /** @@ -262,11 +263,7 @@ export class InboundHandler { data: { status: 'WAITING_ON_CUSTOMER' }, }); } - } else if ( - ticket.status === 'WAITING_ON_CUSTOMER' || - ticket.status === 'RESOLVED' || - ticket.status === 'CLOSED' - ) { + } else if (reopensOnCustomerReply(ticket.status)) { // Customer/external reply reopens a dormant ticket so a human sees it. await this.prisma.ticket.update({ where: { id: ticket.id }, From 202fc775acabd3c876ecdf3bd7aa031aed68f9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:35:07 -0400 Subject: [PATCH 61/83] fix(inbound): build and look up Ticket.sourceId with one shared helper Create stored the ticket's sourceId one way and reply looked it up another: `message.threadId ?? null` on create vs `message.threadId ?? ''` on lookup. With no threadId the two could never agree, so every message in that conversation looked like a brand-new ticket and drew its own AI answer -- defeating the one-answer-per-ticket rule this branch enforces. The Slack composite key was asymmetric the same way (create required BOTH channelId and threadId; the lookup built "channelId:" from channelId alone), and apps/slack-bot/src/events/message.ts built a third variant, "C123:undefined". Root cause was three-way duplication of the key-building logic, so the fix is one definition -- shared/src/platforms/source-id.ts's buildTicketSourceId(source, threadId, channelId) -- that both the writer and every reader derive the key from. findTicketBySourceAndThread(source, threadId, channelId) becomes findTicketBySourceId(source, sourceId): the reader takes the finished key and has no key-building code left to disagree with. Deliberate decision for "no threadId": the helper returns null, not a placeholder. A ticket with no thread key cannot be found again by any lookup, so writers store null (the ticket is still created -- dropping a report is worse) and readers treat null as not-lookup-able and skip the query entirely rather than searching for '' or "C123:", which can only be a false miss or a false hit on a malformed row. Slack with no channelId is also null now instead of a bare thread_ts: a thread_ts is only unique within a channel, and both Slack post paths already threw for those tickets because ticket.channel is null in exactly that case. Documented in the module's doc comment along with the "never inline `${channelId}:${threadId}`" rule. Tests, red-green verified: reverting inbound.ts to the two inline paths gives 5 failed / 37 passed, widened to 8 failed / 38 passed once the unaddressable rows join the write/read symmetry table; breaking the helper to return placeholders gives 3 failed / 3 passed in the new unit test; restored, all green. Covers the previously-untested Slack empty-threadId lookup arm, the Slack no-channelId arms, and the no-threadId case for a non-Slack source (create stores null, reply issues no query, and a threadId-less reply is not matched against a sourceId: null ticket). Verified: packages/outpost `vitest run` 984 passed / `tsc -p shared/tsconfig.json --noEmit` clean; slack-bot 52, discord-bot 57, teams-bot 56, github-app 37 passed; `tsc --noEmit` clean in all four apps. eslint cannot run repo-wide (no flat eslint.config.*) -- pre-existing. Out of scope, untouched: the (source, sourceId) uniqueness constraint / any Prisma migration, and the Slack subtype filter -- both filed as separate follow-ups. Call-Site Enumeration --------------------- ADDED buildTicketSourceId -- grep -rn "buildTicketSourceId" --include="*.ts", all 7 references: - shared/platforms/inbound.ts handleNewTicket: holds. Wants the key to store; null is a valid Ticket.sourceId (nullable column, PrismaLike.create already types it string|null). - shared/platforms/inbound.ts handleReply: holds. Wants the key to search for; narrows null before calling findTicketBySourceId(...: string). - shared/platforms/index.ts re-export: holds. Value export from a module that imports only ../types.js, so no adapter runtime is pulled in. - shared/index.ts re-export: holds. Same reasoning -- the barrel's "platform types only, browser-safe" contract is preserved; justified inline. - apps/slack-bot/src/events/message.ts: holds. source is always TicketSource.SLACK there (SlackAdapter.parseInboundEvent sets it), so the Slack arm applies; a null key returns early instead of querying. - apps/slack-bot/src/lib/tickets.ts findTicketByThreadTs: holds. Its callers (events/actions.ts:22,:74, commands/assign.ts) were written against findFirst and already treat null as "not found", so an early null is indistinguishable. - shared/__tests__/platforms-source-id.test.ts: holds -- unit tests of it. REMOVED findTicketBySourceAndThread -- 2 hits before (definition + its single call, both in inbound.ts), 0 hits repo-wide after. It was private, so no external consumer was even representable. Negative finding: nothing dangles. Near-miss namesake apps/github-app/src/lib/tickets.ts:11 exports a different findTicketBySourceId(sourceId) -- separate package, not imported here, not touched, no collision. CHANGED the sourceId value written on create -- every writer/reader in the repo: - apps/web/.../webhooks/postmark/route.ts:112 writes body.MessageID: holds. Not routed through InboundHandler, and the helper is the identity for EMAIL, so the formats already agree if it ever is. - apps/discord-bot/src/lib/shadow-mode.ts:76 writes thread.id: holds -- identical to buildTicketSourceId(DISCORD, thread.id). - apps/discord-bot/src/lib/tickets.ts:10 and apps/teams-bot/src/lib/tickets.ts:10 read the bare threadId/conversationId: holds. Non-Slack keys are verbatim, so these match what create stores. Left inline deliberately -- for non-Slack the builder is the identity function, there is no format to duplicate. - apps/github-app/src/lib/tickets.ts:11,28 read a prebuilt owner/repo#n: holds, identity for GITHUB_ISSUE/GITHUB_DISCUSSION, unchanged. - shared/platforms/slack.ts:178 extractThreadTs splits on the first ':': holds. The composite format is byte-identical; only would-be bare Slack keys became null, and those tickets already threw on both post paths. - shared/platforms/github.ts:418 parseSourceId regex: holds, GitHub keys unchanged. queue/handlers/github-reaction-poll.ts:22,87 likewise, and it already null-guards ticket.sourceId. - queue/handlers/ai-response.ts:276 passes ticket.sourceId to the adapter: holds -- already string|null per PlatformAdapter.postResponse, and adapters throw loudly on null, the correct outcome for an unaddressable ticket. - shared/platforms/{discord,teams,email-postmark}.ts post paths: hold. All already null-guard sourceId and throw a clear error -- no silent misroute. - TicketSource is still imported in inbound.ts (used by toPlatformTarget), so the import is not dead; confirmed by clean tsc. (cherry picked from commit 33fd91ca68ca1387f29d87e30d0a8afec79a626c) --- apps/slack-bot/src/__tests__/message.test.ts | 19 +++ apps/slack-bot/src/__tests__/tickets.test.ts | 11 ++ apps/slack-bot/src/events/message.ts | 16 ++- apps/slack-bot/src/lib/tickets.ts | 13 +- .../src/__tests__/platforms-inbound.test.ts | 133 ++++++++++++++++++ .../src/__tests__/platforms-source-id.test.ts | 55 ++++++++ packages/outpost/shared/src/index.ts | 5 + .../outpost/shared/src/platforms/inbound.ts | 49 ++++--- .../outpost/shared/src/platforms/index.ts | 4 + .../outpost/shared/src/platforms/source-id.ts | 49 +++++++ 10 files changed, 324 insertions(+), 30 deletions(-) create mode 100644 packages/outpost/shared/src/__tests__/platforms-source-id.test.ts create mode 100644 packages/outpost/shared/src/platforms/source-id.ts diff --git a/apps/slack-bot/src/__tests__/message.test.ts b/apps/slack-bot/src/__tests__/message.test.ts index bb29fa0f..2fefa492 100644 --- a/apps/slack-bot/src/__tests__/message.test.ts +++ b/apps/slack-bot/src/__tests__/message.test.ts @@ -261,6 +261,25 @@ describe('registerMessageHandler', () => { }); }); + it('does not query by a bare thread_ts when the reply event has no channel', async () => { + // The tracked-thread guard used to fall back to the bare thread_ts + // (and to "C123:undefined" when the ts was missing) — keys no Slack + // ticket is ever stored under. Without a channel there is no + // addressable key, so the reply must be dropped, not looked up. + await messageHandler({ + event: { + user: 'U_EXTERNAL', + text: 'Reply with no channel', + ts: '1234567891.000000', + thread_ts: '1234567890.123456', + }, + }); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + expect(prisma.message.create).not.toHaveBeenCalled(); + expect(prisma.ticket.create).not.toHaveBeenCalled(); + }); + it('ignores threaded replies in untracked threads', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); diff --git a/apps/slack-bot/src/__tests__/tickets.test.ts b/apps/slack-bot/src/__tests__/tickets.test.ts index 3b125dab..7c178d97 100644 --- a/apps/slack-bot/src/__tests__/tickets.test.ts +++ b/apps/slack-bot/src/__tests__/tickets.test.ts @@ -46,6 +46,17 @@ describe('findTicketByThreadTs', () => { const result = await findTicketByThreadTs('C_CHAN', '9999999999.000000'); expect(result).toBeNull(); }); + + it('skips the query entirely when channel or ts is missing', async () => { + // buildTicketSourceId yields no key, and no ticket can be stored under + // "C_CHAN:" or ":ts" — querying for one would only ever be a false miss + // (or, worse, a false hit on some other malformed row). + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + expect(await findTicketByThreadTs('C_CHAN', '')).toBeNull(); + expect(await findTicketByThreadTs('', '1234567890.123456')).toBeNull(); + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); }); describe('isTeamMember', () => { diff --git a/apps/slack-bot/src/events/message.ts b/apps/slack-bot/src/events/message.ts index db5b25d6..25c0d01a 100644 --- a/apps/slack-bot/src/events/message.ts +++ b/apps/slack-bot/src/events/message.ts @@ -1,7 +1,7 @@ import type { App } from '@slack/bolt'; import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; -import { SlackAdapter, InboundHandler } from '@copilotkit/outpost/shared/platforms'; +import { SlackAdapter, InboundHandler, buildTicketSourceId } from '@copilotkit/outpost/shared/platforms'; import type { InboundPrismaLike, CreateJobFn } from '@copilotkit/outpost/shared'; import { config } from '../config.js'; @@ -39,9 +39,17 @@ export function registerMessageHandler(app: App): void { // For threaded replies, ignore if the thread isn't tracked as a ticket. // This prevents InboundHandler from creating a new ticket for stray replies. if (!message.isThreadStart) { - const sourceId = message.channelId - ? `${message.channelId}:${message.threadId}` - : message.threadId ?? ''; + // Same key builder InboundHandler writes and reads with — this + // used to build "C123:undefined" for a reply with no threadId, + // a third spelling of a key nothing was ever stored under. + const sourceId = buildTicketSourceId( + message.source, + message.threadId, + message.channelId, + ); + // No addressable key: no ticket can carry it, so this reply is + // untracked by definition. + if (sourceId === null) return; const existingTicket = await prisma.ticket.findFirst({ where: { source: 'SLACK', sourceId }, }); diff --git a/apps/slack-bot/src/lib/tickets.ts b/apps/slack-bot/src/lib/tickets.ts index 36f5f924..fdf602d8 100644 --- a/apps/slack-bot/src/lib/tickets.ts +++ b/apps/slack-bot/src/lib/tickets.ts @@ -1,16 +1,23 @@ import { prisma } from '@copilotkit/outpost/db'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; import { config } from '../config.js'; /** * Find a ticket by its Slack thread timestamp and channel ID. - * Tickets from Slack use a composite sourceId of "channelId:threadTs" - * so we can distinguish threads across channels. + * + * The composite "channelId:threadTs" key is built by buildTicketSourceId — the + * same helper InboundHandler stores tickets with — so this lookup can never + * search for a spelling nothing was written under. A null key (missing channel + * or ts) means no ticket can carry it, so there is nothing to query. */ export async function findTicketByThreadTs(channelId: string, threadTs: string) { + const sourceId = buildTicketSourceId(TicketSource.SLACK, threadTs, channelId); + if (sourceId === null) return null; + return prisma.ticket.findFirst({ where: { source: 'SLACK', - sourceId: `${channelId}:${threadTs}`, + sourceId, }, }); } diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index ef915e4e..0103390a 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -503,6 +503,139 @@ describe('InboundHandler', () => { // Slack tickets use composite sourceId so reply lookups match expect(ticketData.sourceId).toBe('C0ABCDEF1:1234567890.123456'); }); + + it('stores null instead of a bare threadTs when the Slack channelId is missing', async () => { + const msg = makeInboundMessage({ + source: TicketSource.SLACK, + threadId: '1234567890.123456', + channelId: undefined, + isThreadStart: true, + }); + + await handler.handle(msg); + + const ticketData = (prisma.ticket.create as ReturnType).mock.calls[0][0].data; + // A bare ts is not a Slack key — the reply lookup would build + // "channel:ts" and never find it, so refuse to pretend otherwise. + expect(ticketData.sourceId).toBeNull(); + }); + + it('does not search for the unmatchable "channelId:" key when a Slack reply has no threadTs', async () => { + const msg = makeInboundMessage({ + source: TicketSource.SLACK, + threadId: undefined, + channelId: 'C0ABCDEF1', + isThreadStart: false, + }); + + await handler.handle(msg); + + // The old lookup defaulted threadId to '' and queried "C0ABCDEF1:", + // a key nothing is ever stored under. + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); + + it('does not search by a bare threadTs when a Slack reply has no channelId', async () => { + const msg = makeInboundMessage({ + source: TicketSource.SLACK, + threadId: '1234567890.123456', + channelId: undefined, + isThreadStart: false, + }); + + await handler.handle(msg); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); + }); + + // ── sourceId write/read symmetry ───────────────────────────────── + // + // The regression these guard: create stored `null` for a message with no + // threadId while the reply lookup searched for `''`. The two could never + // agree, so every reply in such a conversation looked like a brand-new + // ticket and drew its own AI answer. + + describe('sourceId write/read symmetry', () => { + it('does not search for the empty-string key when a non-Slack reply has no threadId', async () => { + const msg = makeInboundMessage({ + source: TicketSource.DISCORD, + threadId: undefined, + isThreadStart: false, + }); + + await handler.handle(msg); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); + + it('stores null for a non-Slack ticket with no threadId', async () => { + const msg = makeInboundMessage({ + source: TicketSource.DISCORD, + threadId: undefined, + isThreadStart: true, + }); + + await handler.handle(msg); + + const ticketData = (prisma.ticket.create as ReturnType).mock.calls[0][0].data; + expect(ticketData.sourceId).toBeNull(); + }); + + it('treats a threadId-less reply as a new ticket instead of matching a null-sourceId ticket', async () => { + // Guard against the opposite failure mode: matching on the absent + // key would glue unrelated threadId-less conversations together. + (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ + id: 'ticket-existing', + displayId: 'TKT-EXISTIN', + status: 'OPEN', + sourceId: null, + channel: 'channel-1', + source: 'DISCORD', + }); + + const msg = makeInboundMessage({ + source: TicketSource.DISCORD, + threadId: undefined, + isThreadStart: false, + }); + const result = await handler.handle(msg); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + expect(result.isNewTicket).toBe(true); + }); + + it.each([ + [TicketSource.DISCORD, 'thread-abc', 'channel-1'], + [TicketSource.SLACK, '1234567890.123456', 'C0ABCDEF1'], + [TicketSource.TEAMS, 'conv-xyz', undefined], + [TicketSource.GITHUB_ISSUE, 'owner/repo#42', undefined], + // The unaddressable cases: written key is null, so no lookup may + // happen at all. Any query here means the reader invented a key. + [TicketSource.DISCORD, undefined, 'channel-1'], + [TicketSource.TEAMS, undefined, undefined], + [TicketSource.SLACK, '1234567890.123456', undefined], + [TicketSource.SLACK, undefined, 'C0ABCDEF1'], + ])( + 'writes and reads the same key for %s (threadId=%s, channelId=%s)', + async (source, threadId, channelId) => { + await handler.handle( + makeInboundMessage({ source, threadId, channelId, isThreadStart: true }), + ); + const written = (prisma.ticket.create as ReturnType).mock.calls[0][0] + .data.sourceId; + + const findFirst = prisma.ticket.findFirst as ReturnType; + await handler.handle( + makeInboundMessage({ source, threadId, channelId, isThreadStart: false }), + ); + // No query at all is the correct read of a null key. + const read = + findFirst.mock.calls.length === 0 ? null : findFirst.mock.calls[0][0].where.sourceId; + + expect(read).toBe(written); + }, + ); }); // ── Team member detection ──────────────────────────────────────── diff --git a/packages/outpost/shared/src/__tests__/platforms-source-id.test.ts b/packages/outpost/shared/src/__tests__/platforms-source-id.test.ts new file mode 100644 index 00000000..f6d7d080 --- /dev/null +++ b/packages/outpost/shared/src/__tests__/platforms-source-id.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from 'vitest'; +import { buildTicketSourceId } from '../platforms/source-id.js'; +import { TicketSource } from '../types.js'; + +describe('buildTicketSourceId', () => { + it('returns the threadId verbatim for non-Slack sources', () => { + expect(buildTicketSourceId(TicketSource.DISCORD, 'thread-123', 'channel-1')).toBe('thread-123'); + expect(buildTicketSourceId(TicketSource.TEAMS, 'conv-abc')).toBe('conv-abc'); + expect(buildTicketSourceId(TicketSource.GITHUB_ISSUE, 'owner/repo#42')).toBe('owner/repo#42'); + }); + + it('builds the composite channelId:threadId key for Slack', () => { + expect(buildTicketSourceId(TicketSource.SLACK, '1234567890.123456', 'C0ABCDEF1')).toBe( + 'C0ABCDEF1:1234567890.123456', + ); + }); + + // The whole point of the helper: no input may produce a key that only one + // of the two call sites would ever build. + it('returns null rather than a placeholder when there is no threadId', () => { + expect(buildTicketSourceId(TicketSource.DISCORD, undefined, 'channel-1')).toBeNull(); + expect(buildTicketSourceId(TicketSource.DISCORD, '', 'channel-1')).toBeNull(); + expect(buildTicketSourceId(TicketSource.DISCORD, null)).toBeNull(); + // Never the empty-string key the old lookup searched for. + expect(buildTicketSourceId(TicketSource.DISCORD, '')).not.toBe(''); + }); + + it('returns null for Slack when the channelId is missing', () => { + // A Slack thread_ts is only unique within a channel, and posting the + // reply needs the channel anyway — a bare ts is not a usable key. + expect(buildTicketSourceId(TicketSource.SLACK, '1234567890.123456')).toBeNull(); + expect(buildTicketSourceId(TicketSource.SLACK, '1234567890.123456', '')).toBeNull(); + }); + + it('never emits the unmatchable "channel:" or "channel:undefined" Slack keys', () => { + expect(buildTicketSourceId(TicketSource.SLACK, '', 'C123')).toBeNull(); + expect(buildTicketSourceId(TicketSource.SLACK, undefined, 'C123')).toBeNull(); + }); + + it('is deterministic — the same inputs always give the writer and reader the same key', () => { + const cases: Array<[TicketSource, string | undefined, string | undefined]> = [ + [TicketSource.SLACK, '111.222', 'C1'], + [TicketSource.SLACK, '111.222', undefined], + [TicketSource.DISCORD, 'thread-9', 'C1'], + [TicketSource.DISCORD, undefined, 'C1'], + [TicketSource.EMAIL, 'msg-1@postmark', undefined], + ]; + + for (const [source, threadId, channelId] of cases) { + expect(buildTicketSourceId(source, threadId, channelId)).toBe( + buildTicketSourceId(source, threadId, channelId), + ); + } + }); +}); diff --git a/packages/outpost/shared/src/index.ts b/packages/outpost/shared/src/index.ts index 3d006123..d4ab0fca 100644 --- a/packages/outpost/shared/src/index.ts +++ b/packages/outpost/shared/src/index.ts @@ -9,6 +9,11 @@ export * from './integrations/index.js'; export * from './monitoring/index.js'; export * from './sync/index.js'; export * from './auth/index.js'; +// buildTicketSourceId is the one exception to the "types only" rule below: it +// is a pure string function importing nothing but the TicketSource enum, so it +// stays browser-safe while giving every consumer (bots, InboundHandler, web) +// a single definition of the Ticket.sourceId key. +export { buildTicketSourceId } from './platforms/source-id.js'; // Platform types only — safe for browser bundling (no runtime adapter code). // Bot apps that need the actual adapter classes should import from // '@copilotkit/outpost/shared/platforms' instead. diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 096be070..3e1b8c47 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -15,6 +15,7 @@ import type { InboundMessage, InboundResult, TicketRef } from './types.js'; import { generateTicketId, truncate } from '../utils.js'; import { reopensOnCustomerReply } from '../constants.js'; import { TicketSource } from '../types.js'; +import { buildTicketSourceId } from './source-id.js'; /** * Prisma client interface — the subset of PrismaClient we actually call. @@ -139,12 +140,16 @@ export class InboundHandler { const displayId = generateTicketId(); const authorLabel = `${message.platformUsername} (${message.platformUserId})`; - // Build sourceId — Slack uses a composite "channelId:threadTs" key - // so that reply lookups match the same format. - let sourceId = message.threadId ?? null; - if (message.source === TicketSource.SLACK && message.channelId && message.threadId) { - sourceId = `${message.channelId}:${message.threadId}`; - } + // Build sourceId through the SAME helper handleReply's lookup uses, so + // the stored key and the searched-for key cannot drift apart. null here + // means "this thread is not addressable" (no threadId, or Slack with no + // channelId) — the ticket is still created so the report is not dropped, + // but it will never be matched by a later reply. + const sourceId = buildTicketSourceId( + message.source, + message.threadId, + message.channelId, + ); // Find-or-create the User row for the message sender so the ticket // can be linked to them (needed for reporter-identity lookups like @@ -213,13 +218,19 @@ export class InboundHandler { * Handle a reply to an existing ticket thread. */ private async handleReply(message: InboundMessage): Promise { - // Look up the existing ticket by source + threadId - const ticket = await this.findTicketBySourceAndThread( + // Derive the lookup key with the same helper handleNewTicket stores + // with. A null key means no ticket could ever carry it, so skip the + // query entirely rather than searching for a synthesized placeholder. + const sourceId = buildTicketSourceId( message.source, - message.threadId ?? '', + message.threadId, message.channelId, ); + const ticket = sourceId === null + ? null + : await this.findTicketBySourceId(message.source, sourceId); + if (!ticket) { // No existing ticket found for this thread — treat as a new ticket. // This handles edge cases where a reply arrives before the thread-start @@ -281,24 +292,16 @@ export class InboundHandler { } /** - * Find an existing ticket by its source platform and thread/conversation ID. + * Find an existing ticket by source platform + an already-built sourceId. * - * For Slack, the sourceId is "channelId:threadTs" so we use channelId - * to reconstruct the composite key. For other platforms, sourceId is - * the threadId directly. + * Deliberately takes the finished key rather than (threadId, channelId): + * key construction lives in buildTicketSourceId alone, so this method + * cannot disagree with what handleNewTicket stored. */ - private async findTicketBySourceAndThread( + private async findTicketBySourceId( source: TicketSource, - threadId: string, - channelId?: string, + sourceId: string, ): Promise { - let sourceId = threadId; - - // Slack uses a composite sourceId: "channelId:threadTs" - if (source === TicketSource.SLACK && channelId) { - sourceId = `${channelId}:${threadId}`; - } - const ticket = await this.prisma.ticket.findFirst({ where: { source: source as string, diff --git a/packages/outpost/shared/src/platforms/index.ts b/packages/outpost/shared/src/platforms/index.ts index e0342f52..adfef45f 100644 --- a/packages/outpost/shared/src/platforms/index.ts +++ b/packages/outpost/shared/src/platforms/index.ts @@ -32,6 +32,10 @@ export { SUPPORTED_PLATFORMS, } from './registry.js'; +// Ticket sourceId key builder — the single definition shared by ticket +// creation, reply lookup, and the bots' own "is this thread tracked?" checks. +export { buildTicketSourceId } from './source-id.js'; + // Inbound handler export { InboundHandler } from './inbound.js'; export type { InboundHandlerConfig, CreateJobFn } from './inbound.js'; diff --git a/packages/outpost/shared/src/platforms/source-id.ts b/packages/outpost/shared/src/platforms/source-id.ts new file mode 100644 index 00000000..c0c10dc2 --- /dev/null +++ b/packages/outpost/shared/src/platforms/source-id.ts @@ -0,0 +1,49 @@ +/** + * Ticket sourceId key construction — ONE definition, used by both writers + * and readers. + * + * `Ticket.sourceId` is the platform-thread key Outpost uses to decide whether + * an inbound message opens a new ticket or belongs to an existing one. It is + * written once (on ticket create) and read on every reply. Those two sites MUST + * derive the key identically or every reply looks like a brand-new ticket and + * gets its own AI answer — the exact bug this module exists to make + * unrepresentable. Do not inline `${channelId}:${threadId}` anywhere; call + * `buildTicketSourceId`. + */ + +import { TicketSource } from '../types.js'; + +/** + * Build the `Ticket.sourceId` key for a platform thread, or return `null` when + * this message cannot produce a thread-addressable key. + * + * Rules: + * - No `threadId` → `null`. A ticket with no thread key cannot be found again + * by any lookup, so we refuse to synthesize one (the old code stored `null` + * on create but searched for `''` on reply, so the two could never agree). + * Callers writing a ticket store `null`; callers reading treat `null` as + * "not lookup-able" and must not query. + * - Slack → composite `"channelId:threadId"`. A Slack `thread_ts` is only + * unique within a channel, and `SlackAdapter.postResponse` needs the channel + * anyway, so a Slack message with no `channelId` yields `null` rather than a + * bare `threadId` that could collide across channels. + * - Everything else → the `threadId` verbatim (Discord thread ID, Teams + * conversation ID, `owner/repo#number` for GitHub, Postmark MessageID). + * + * Empty strings count as absent — platform adapters default missing IDs to + * `''` (see `SlackAdapter.parseInboundEvent`), and `''` is not a usable key. + */ +export function buildTicketSourceId( + source: TicketSource, + threadId?: string | null, + channelId?: string | null, +): string | null { + if (!threadId) return null; + + if (source === TicketSource.SLACK) { + if (!channelId) return null; + return `${channelId}:${threadId}`; + } + + return threadId; +} From 0052d36f6ec69aa60b7408e3ba7d62d6fc2d7ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:38:55 -0400 Subject: [PATCH 62/83] fix(inbound): never answer an orphaned reply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleReply stopped enqueuing AI_RESPONSE, but on a ticket-lookup MISS it fell back to `handleNewTicket({ ...message, isThreadStart: true })` — and handleNewTicket enqueues. A mid-thread reply became a brand-new "ticket" titled with the follow-up text, and the bot answered it. The one-answer-per- ticket rule was routed around by the same file that documents it. Reachability was not theoretical: - apps/slack-bot/src/events/message.ts pre-filters replies whose thread is untracked, so Slack was shielded. - apps/teams-bot/src/handlers/message.ts has NO such pre-filter and its monitored-channel gate only runs for thread starts, so a Teams reply reached the fallback. - Discord and GitHub reached it for any thread predating Outpost. handleNewTicket now takes an explicit `{ answer: boolean }` decision from its caller instead of inferring one. handle() passes `{ answer: true }` for a genuine thread start; the orphaned-reply fallback passes `{ answer: false }`. No duplication of handleNewTicket's body, and InboundResult.aiJobEnqueued stays truthful (false on that path). ASSUMPTION, stated in the code comment so it is reviewable: an orphaned reply still CREATES a ticket and persists the message — dropping a customer's message is worse than filing an oddly-titled ticket — but it is never answered, because the message that opened the real conversation was never seen by us. A human picks it up from the dashboard. Also corrects the now-falsified claim in queue/src/handlers/ai-response.ts that the ticket-history gate makes the invariant "unroutable-around". That gate only sees messages on the ticket, so a ticket freshly minted around a mid-thread message has no prior AI response and sails through it. The gate is a re-answer guard, not a total gate; the orphaned-reply case must be refused at the enqueue site, and the comment now says so and points at it. Tests: both pre-existing fallback tests asserted nothing about createJob, which is exactly why this shipped. Both now assert it, plus new coverage for Teams/Discord/GitHub/Slack orphaned replies, a non-team-member sender (the case that WOULD have been answered), and a genuine thread start still being answered so the fix is not a blanket mute. Red-green verified: with `{ answer: false }` flipped to `{ answer: true }`, 7 tests fail in packages/outpost (2 files) and 1 in apps/teams-bot; restored, all green. Call-Site Enumeration --------------------- Symbols changed: `InboundHandler.handleNewTicket` (signature — added required second param `{ answer: boolean }`). Symbols added: none exported. Symbols removed: none. `handleNewTicket` — private; grep over the repo (excluding node_modules) returns exactly three hits, all in shared/src/platforms/inbound.ts: - :147 the declaration. - :133 `handle()` thread-start branch → passes `{ answer: true }`. Assumption holds: isThreadStart=true is the opening message, the one message Outpost may answer. Behavior byte-identical to before. - :252 `handleReply` orphan fallback → passes `{ answer: false }`. Assumption deliberately INVERTED here; that is the fix. No external caller exists and none can be added by accident: the method is `private` and is not re-exported from platforms/index.ts or shared/index.ts (verified — those export the class, InboundHandlerConfig, CreateJobFn only). `handleReply` — private; hits at :135 (sole caller, the non-thread-start branch of handle()) and :226 (declaration). Signature unchanged; its return type and every non-orphan path are untouched. Third hit is a comment reference in queue/src/handlers/ai-response.ts:100 (prose, no call). `InboundResult.aiJobEnqueued` — non-test consumers: - shared/src/platforms/types.ts:165 — the declaration, `boolean`, unchanged. - apps/discord-bot/src/events/message-create.ts:68 — logs " (AI job enqueued)" when true. Assumption still holds and is now MORE accurate: on an orphaned Discord reply the log no longer claims an enqueue that would have happened. Log text only, no control flow. NEGATIVE FINDING: no other runtime code reads aiJobEnqueued — not the Slack, Teams, or GitHub bots, not the queue, not the dashboard. grep for the identifier over apps/ and packages/ returns only the above plus test files. So no caller branches on it and no caller can be broken by it flipping to false on this path. `isTeamMember` — still called exactly where it was, now guarded by `answer &&` short-circuit so the lookup is skipped when no answer is possible. NEGATIVE FINDING: the returned value was used for nothing but the enqueue decision in handleNewTicket (the reply path calls it separately for its own status transitions, untouched), so skipping the query cannot change any other observable behavior. The reply path's own isTeamMember call at :281 is unaffected. `handle` (public entry point) — signature and return type unchanged; all bot call sites (apps/discord-bot thread-create.ts + message-create.ts, apps/slack-bot events/message.ts, apps/teams-bot handlers/message.ts, apps/github-app webhooks/issues-opened.ts + discussion-created.ts) compile and pass unchanged. NEGATIVE FINDING: no bot needed a per-platform change; the fix is entirely in the shared handler, so all four platforms are covered at once and no bot can opt out. `ai-response.ts` change — comments only, zero code touched, so it has no call sites and no type surface. Verification ------------ - packages/outpost: `npx vitest run --reporter=dot` → 60 files, 972 tests, all passing. (`npx prisma generate` was needed first in this fresh worktree; without it queue/src/__tests__/scheduler.test.ts fails to load on a missing .prisma/client — pre-existing env setup, not a code failure.) - `npx tsc --project shared/tsconfig.json --noEmit` → clean. - apps/teams-bot 57, apps/discord-bot 57, apps/slack-bot 50, apps/github-app 37 — all passing. (cherry picked from commit 6f14b00703f954280922af044bc2500337ef41f9) --- .../src/__tests__/inbound-handler.test.ts | 22 +++++++ .../outpost/queue/src/handlers/ai-response.ts | 10 ++- .../src/__tests__/inbound-handler.test.ts | 14 +++++ .../src/__tests__/platforms-inbound.test.ts | 62 +++++++++++++++++++ .../outpost/shared/src/platforms/inbound.ts | 56 ++++++++++++----- 5 files changed, 148 insertions(+), 16 deletions(-) diff --git a/apps/teams-bot/src/__tests__/inbound-handler.test.ts b/apps/teams-bot/src/__tests__/inbound-handler.test.ts index 4f124e31..aae13b2a 100644 --- a/apps/teams-bot/src/__tests__/inbound-handler.test.ts +++ b/apps/teams-bot/src/__tests__/inbound-handler.test.ts @@ -233,4 +233,26 @@ describe('InboundHandler (Teams-focused)', () => { expect(prisma.ticket.update).not.toHaveBeenCalled(); }); }); + + // Teams is the platform where this is most reachable: unlike the Slack bot, + // apps/teams-bot/src/handlers/message.ts has no untracked-thread pre-filter, + // and its monitored-channel gate only runs for thread starts. So a reply in + // a Teams conversation Outpost never saw arrives here with no ticket. + describe('orphaned reply (no ticket for the conversation)', () => { + beforeEach(() => { + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + }); + + it('files a ticket for a human but never answers', async () => { + const result = await handler.handle( + makeMessage({ isThreadStart: false, content: 'thanks, that worked!' }), + ); + + expect(prisma.ticket.create).toHaveBeenCalledTimes(1); + expect(prisma.message.create).toHaveBeenCalledTimes(1); + expect(createJob).not.toHaveBeenCalled(); + expect(result.aiJobEnqueued).toBe(false); + expect(result.isNewTicket).toBe(true); + }); + }); }); diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 031190e6..45a32547 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -89,7 +89,15 @@ export async function handleAiResponse( // each one previously decided for itself whether a reply warranted an // answer. Those enqueues are gone, but a single new caller added later // would silently reintroduce the follow-up spam this closes. Checking the - // ticket's own history instead makes the invariant unroutable-around. + // ticket's own history catches every re-answer of a ticket we already + // answered. + // + // It is NOT a total gate, so do not lean on it as one. It can only see + // messages on the ticket, so it cannot tell a first answer from a first + // answer to the wrong message: a ticket freshly minted around a mid-thread + // message has no prior AI response and would sail through here. That case + // (an orphaned reply, no ticket found for the thread) is refused at the + // enqueue site in InboundHandler.handleReply — see the comment there. // // Success, not failure: the job did what it should — nothing. Returning an // error would put it through the retry ladder for a decision that will diff --git a/packages/outpost/shared/src/__tests__/inbound-handler.test.ts b/packages/outpost/shared/src/__tests__/inbound-handler.test.ts index a8a44fc7..42708e66 100644 --- a/packages/outpost/shared/src/__tests__/inbound-handler.test.ts +++ b/packages/outpost/shared/src/__tests__/inbound-handler.test.ts @@ -227,6 +227,20 @@ describe('InboundHandler (GitHub-focused)', () => { expect(prisma.ticket.create).toHaveBeenCalledTimes(1); }); + // An issue comment on an issue that predates Outpost lands here: no + // ticket exists for the thread. File it for a human, but never answer — + // the issue body (the message that opened the conversation) was never + // seen by us, and Outpost answers only the opening message. + it('files the orphaned comment but enqueues NO AI_RESPONSE', async () => { + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + const result = await handler.handle(makeFollowUpMessage()); + + expect(prisma.message.create).toHaveBeenCalledTimes(1); + expect(createJob).not.toHaveBeenCalled(); + expect(result.aiJobEnqueued).toBe(false); + }); + it('returns isNewTicket=false for follow-ups', async () => { const message = makeFollowUpMessage(); const result = await handler.handle(message); diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index 0103390a..88a8d660 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -468,6 +468,68 @@ describe('InboundHandler', () => { }); }); + // ── Orphaned replies (reply with no matching ticket) ───────────── + // + // Outpost answers exactly ONE message per ticket: the one that OPENED it. + // An orphaned reply is mid-conversation, so we file it (never drop a + // customer's words) but must not answer it — the opening message was never + // seen by us. These assertions exist because the pre-existing fallback + // tests asserted nothing about createJob, which is how a regression here + // shipped: the fallback re-entered the new-ticket path and answered. + describe('orphaned reply fallback never answers', () => { + beforeEach(() => { + (prisma.ticket.findFirst as ReturnType).mockResolvedValue(null); + }); + + it('creates the ticket and message but enqueues NO AI_RESPONSE', async () => { + const msg = makeInboundMessage({ + isThreadStart: false, + content: 'any update on this?', + }); + const result = await handler.handle(msg); + + expect(prisma.ticket.create).toHaveBeenCalledTimes(1); + expect(prisma.message.create).toHaveBeenCalledTimes(1); + expect(createJob).not.toHaveBeenCalled(); + expect(result.aiJobEnqueued).toBe(false); + expect(result.isNewTicket).toBe(true); + expect(result.messageId).toBe('msg-1'); + }); + + it('does not answer even when the sender is not a team member', async () => { + // Non-team sender is the case that WOULD have been answered by the + // new-ticket path — the exact hole this closes. + (prisma.user.findFirst as ReturnType).mockResolvedValue(null); + (prisma.teamMember.findUnique as ReturnType).mockResolvedValue(null); + + const result = await handler.handle(makeInboundMessage({ isThreadStart: false })); + + expect(createJob).not.toHaveBeenCalled(); + expect(result.aiJobEnqueued).toBe(false); + }); + + it.each([ + [TicketSource.TEAMS, 'teams'], + [TicketSource.DISCORD, 'discord'], + [TicketSource.GITHUB_ISSUE, 'github'], + [TicketSource.SLACK, 'slack'], + ])('stays silent for an orphaned %s reply', async (source) => { + const result = await handler.handle( + makeInboundMessage({ source, isThreadStart: false, channelId: 'chan-1' }), + ); + + expect(createJob).not.toHaveBeenCalled(); + expect(result.aiJobEnqueued).toBe(false); + }); + + it('still enqueues for a genuine thread start, so the fix is not a blanket mute', async () => { + const result = await handler.handle(makeInboundMessage({ isThreadStart: true })); + + expect(createJob).toHaveBeenCalledTimes(1); + expect(result.aiJobEnqueued).toBe(true); + }); + }); + // ── Slack composite sourceId ───────────────────────────────────── describe('Slack composite sourceId handling', () => { diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 3e1b8c47..554d7bfd 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -7,8 +7,11 @@ * 2. Replies (isThreadStart=false): find existing ticket, create Message, reopen if needed. * Never enqueues AI_RESPONSE — Outpost answers once per ticket, on the opening * message only, and a human owns the thread after that. - * 3. Team member detection via ExternalIdentity -> TeamMember lookup - * 4. Sequential display ID generation (TKT-XXXXXXXX) + * 3. Orphaned replies (isThreadStart=false with no matching ticket): create the + * Ticket + Message so the customer's words are never dropped, but do NOT + * enqueue AI_RESPONSE — we never saw the message that opened the conversation. + * 4. Team member detection via ExternalIdentity -> TeamMember lookup + * 5. Sequential display ID generation (TKT-XXXXXXXX) */ import type { InboundMessage, InboundResult, TicketRef } from './types.js'; @@ -122,21 +125,31 @@ export class InboundHandler { /** * Process an inbound message. * - * Determines whether this is a new ticket or a reply to an existing one, - * creates the appropriate database records, and enqueues an AI_RESPONSE - * job if the sender is not a team member. + * Determines whether this is a new ticket or a reply to an existing one and + * creates the appropriate database records. An AI_RESPONSE job is enqueued + * only for a genuine thread start from a non-team-member — never for a + * reply, and never for the orphaned-reply fallback below. */ async handle(message: InboundMessage): Promise { if (message.isThreadStart) { - return this.handleNewTicket(message); + return this.handleNewTicket(message, { answer: true }); } return this.handleReply(message); } /** * Create a new ticket from a thread-start message. + * + * `answer` is an explicit decision made by the caller, never inferred from + * the message: `true` for a genuine thread start (the opening message is + * the one message Outpost is allowed to answer), `false` for the orphaned- + * reply fallback in `handleReply`, where we are creating a ticket around a + * mid-conversation message we must not answer. */ - private async handleNewTicket(message: InboundMessage): Promise { + private async handleNewTicket( + message: InboundMessage, + { answer }: { answer: boolean }, + ): Promise { const displayId = generateTicketId(); const authorLabel = `${message.platformUsername} (${message.platformUserId})`; @@ -192,11 +205,10 @@ export class InboundHandler { messageId = msg.id; } - // Check if sender is a team member — they still get a ticket but skip AI - const isTeam = await this.isTeamMember(message.platformUserId, message.source); - + // Team members still get a ticket but no AI answer. Only consulted when + // the caller allowed an answer at all — otherwise the lookup is wasted. let aiJobEnqueued = false; - if (!isTeam) { + if (answer && !(await this.isTeamMember(message.platformUserId, message.source))) { await this.createJob(this.aiResponseJobType, { ticketId: ticket.id, threadId: message.threadId, @@ -232,10 +244,24 @@ export class InboundHandler { : await this.findTicketBySourceId(message.source, sourceId); if (!ticket) { - // No existing ticket found for this thread — treat as a new ticket. - // This handles edge cases where a reply arrives before the thread-start - // event, or the original ticket was deleted. - return this.handleNewTicket({ ...message, isThreadStart: true }); + // Orphaned reply: a mid-thread message whose thread we have no ticket + // for — the thread predates Outpost, the platform delivered the reply + // before the thread-start event, or the original ticket was deleted. + // + // We still create a ticket and persist the message: dropping a + // customer's words is worse than filing an oddly-titled ticket, and a + // human can pick it up from the dashboard. + // + // We do NOT answer it. ASSUMPTION, stated so it is reviewable: the + // message that opened the real conversation was never seen by us, so + // this reply is not "the message that opened the ticket" in the + // product sense even though it is the ticket's first message. Outpost + // answers exactly one message per ticket — the opening one — and this + // is not it. Answering here is how a follow-up ("any update?", or a + // community member's reply to someone else) used to get an AI reply + // in a thread Outpost was never part of; that routed around the + // one-answer-per-ticket rule entirely. + return this.handleNewTicket({ ...message, isThreadStart: true }, { answer: false }); } const authorLabel = `${message.platformUsername} (${message.platformUserId})`; From 508e904faec6cebc3c3c13b10bccae4bfb460b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:34:18 -0400 Subject: [PATCH 63/83] fix(ai): escalate when an AI answer never reaches the reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one-response-per-ticket guard reads the BOT Message row, which is committed BEFORE the platform post-back. That made two pre-existing soft-failure paths permanent: 1. The `suggestedResponse` ticket.update ran unguarded between the BOT row and post-back. A throw there aborted the job; every retry then hit the guard, returned success with skipped: true, and nothing was posted or escalated. 2. `adapter.postResponse` throwing was logged and the job still reported success. Before the guard a manual re-enqueue could still deliver the answer; after it, that door is closed. Either way the reporter is silent forever while the DB says they were answered. The guard is the requirement, so it is untouched — instead every path where the answer failed to reach the reporter now hands the thread to a human in the same run: - `suggestedResponse` write is non-fatal (logged); it can no longer strand the job before delivery is attempted. - postResponse throwing, and getAdapter throwing, record a deliveryFailure. - deliveryFailure enqueues ESCALATION with a reason naming the failure, and wins the reason slot over suppression / low confidence (most actionable). - If that ESCALATION enqueue also fails, the job returns success: false so the worker records a failed attempt with the reason on the job row — the one outcome with neither delivery nor a human must not look like success. Escalation-enqueue failures for low confidence / suppression keep the historical success result: there the response did reach the reporter. - The externalCommentId write moved out of the post-back try so bookkeeping failure is not misread as delivery failure. - Sources with no adapter escalate only if the suggestedResponse write failed, since there suggestedResponse IS the delivery path. SHADOW_MODE posts nothing by design and never escalates on that basis. Out of scope (filed separately): the guard's check-then-write race under AI_RESPONSE concurrency 4, whose fix is a uniqueness constraint plus a migration. Nothing here narrows or widens that window — no ordering of the guard read or the Message create changed, and no transaction was added. Call-Site Enumeration --------------------- `handleAiResponse` (exported; signature unchanged) - packages/outpost/queue/src/index.ts:4 re-export — holds, same signature. - apps/worker/src/index.ts:82 `worker.on(JobType.AI_RESPONSE, ...)` — holds. The new success: false path is a JobResult the worker already handles at worker.ts:351 via handleFailure (retry ladder + error text persisted on the job row). A retry after that failure is skipped by the guard and returns success, which completes the job; the recorded error text remains on the row, so the operator signal survives. That is intended: the answer is undeliverable, so retrying generation is pointless. - queue/src/__tests__/ai-response.test.ts — updated, all call sites reviewed. - No other callers (grepped `handleAiResponse` across packages/ and apps/). JobResult.data key `escalated` (semantics widened: now also true on delivery failure) - Grepped `escalated` across packages/ and apps/: every hit is unrelated (AI disclaimer copy, escalation handler log lines, bot escalate commands) except ai-response.test.ts. NEGATIVE FINDING: no production reader of AI_RESPONSE's result.data exists — worker.ts only inspects `success` and stores nothing from `data` — so widening it breaks nothing. JobResult.data key `deliveryFailed` (new) - Only readers are the new tests. No dashboard/API code reads AI_RESPONSE job result payloads (grepped `result.data` under queue/src and apps/). ESCALATION payload `reason` (new value shape for the delivery case) - queue/src/handlers/escalation.ts:33,66,108,130 — holds. `reason` is used only as free text: interpolated into the routing reason, the assignment note and a SYSTEM message. No parsing, no enum matching, no length limit. - apps/* escalate commands construct their own reasons; unaffected. New locals (`deliveryFailure`, `escalationEnqueueError`, `suggestedResponseError`, `escalationReason`, `escalated`) and the widened scope of `externalCommentId` - All function-local to handleAiResponse; no exports, no references outside the handler body. `externalCommentId` changed from a `const` inside the post-back try to a `let` in the enclosing block; its only consumer is the message.update below it, which now runs on the delivered path only. Tests ----- Red-green verified for each behaviour: dropping the postResponse deliveryFailure assignment (3 red), restoring the unguarded suggestedResponse write (2 red), disabling the success: false return (1 red), and moving the externalCommentId write back inside the post-back try (1 red). `npx vitest run` — 973 passed (60 files). `npx tsc --project queue/tsconfig.json --noEmit` — clean. (cherry picked from commit 71dd6a9702f1b7811cee7ae2237bfd3233481cb0) --- .../queue/src/__tests__/ai-response.test.ts | 181 ++++++++++++++++++ .../outpost/queue/src/handlers/ai-response.ts | 143 +++++++++++--- 2 files changed, 294 insertions(+), 30 deletions(-) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index 9fd4f959..c27b259e 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -745,6 +745,187 @@ describe('handleAiResponse', () => { expect(mockPostResponse).not.toHaveBeenCalled(); }); + // ── Undelivered responses always end up with a human ────────────────── + // + // The one-response-per-ticket guard reads the BOT Message row, which is + // committed BEFORE the platform post-back. So once generation has happened, + // no retry and no manual re-enqueue can ever deliver that answer — the guard + // skips them all, correctly. The consequence is that every path where the + // answer failed to reach the reporter has to hand the thread to a human + // right here, in this run, or the reporter is silently abandoned while the + // database claims they were answered. + // + // These tests pin that: a delivery failure always produces an ESCALATION + // job, the pre-post-back writes can never abort the job before delivery is + // attempted, and the one outcome with neither delivery nor escalation is + // reported as a job failure instead of a success. + describe('delivery failures escalate to a human', () => { + /** Reject only the suggestedResponse write, not the classification one. */ + function failSuggestedResponseWrite(message: string): void { + mockPrismaTicket.update.mockImplementation( + async (args: { data: Record }) => { + if (args.data.suggestedResponse !== undefined) { + throw new Error(message); + } + return {}; + }, + ); + } + + /** The single ESCALATION job payload, asserting exactly one was created. */ + function escalationPayload(): Record { + const calls = mockPrismaJob.create.mock.calls.filter( + (call: Array<{ data: { type: string } }>) => call[0].data.type === 'ESCALATION', + ); + expect(calls).toHaveLength(1); + return calls[0][0].data.payload as Record; + } + + it('enqueues an ESCALATION job when post-back throws', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data?.escalated).toBe(true); + expect(result.data?.deliveryFailed).toBe(true); + expect(escalationPayload()).toEqual( + expect.objectContaining({ + ticketId: 'tkt-1', + reason: expect.stringContaining('not delivered'), + }), + ); + // The reason has to name the delivery failure so the human picking it + // up knows the answer exists but never landed. + expect(escalationPayload().reason).toContain('Discord API 503'); + }); + + it('enqueues an ESCALATION job when the adapter cannot be constructed', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGetAdapter.mockImplementation(() => { + throw new Error('Missing DISCORD_BOT_TOKEN'); + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data?.deliveryFailed).toBe(true); + expect(escalationPayload().reason).toContain('adapter misconfigured'); + }); + + it('names the delivery failure even when confidence is also low', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(lowConfidenceResult); + mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + // One escalation, and it reports the more actionable of the two facts. + expect(escalationPayload().reason).toContain('not delivered'); + }); + + it('does not escalate a delivered high-confidence response', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.data?.deliveryFailed).toBe(false); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + }); + + it('still attempts post-back when the suggestedResponse write throws', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + failSuggestedResponseWrite('DB write conflict'); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + // The failed write must not abort the job between the BOT row and the + // post-back — that is the window the guard makes unrecoverable. + expect(mockPostResponse).toHaveBeenCalled(); + expect(result.success).toBe(true); + expect(result.data?.deliveryFailed).toBe(false); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + }); + + it('escalates when the suggestedResponse write throws and there is no adapter', async () => { + // With no adapter, suggestedResponse IS the delivery path. + mockPrismaTicket.findUnique.mockResolvedValue({ ...sampleTicket, source: 'WEB' }); + mockHasAdapter.mockReturnValue(false); + failSuggestedResponseWrite('DB write conflict'); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'web' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data?.deliveryFailed).toBe(true); + expect(escalationPayload().reason).toContain('DB write conflict'); + }); + + it('does not treat a failed externalCommentId write as a delivery failure', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockPostResponse.mockResolvedValue('999888'); + mockPrismaMessage.update.mockRejectedValueOnce(new Error('DB write conflict')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + // The response reached the reporter; only the bookkeeping row failed. + expect(result.success).toBe(true); + expect(result.data?.deliveryFailed).toBe(false); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + }); + + it('reports job failure when the answer was neither delivered nor escalated', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); + mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + // Nothing reached the reporter and no human was pulled in; a silent + // success here is exactly the outcome this fix exists to prevent. + expect(result.success).toBe(false); + expect(result.error).toContain('Discord API 503'); + expect(result.error).toContain('queue unavailable'); + }); + + it('still reports success when only a low-confidence escalation fails to enqueue', async () => { + // The reporter did get the answer here, so the historical fail-soft + // behaviour stands — the failure mode is different in kind. + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(lowConfidenceResult); + mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(mockPostResponse).toHaveBeenCalled(); + expect(result.success).toBe(true); + }); + }); + it('succeeds even if shadow mode message logging fails', async () => { const originalShadow = process.env.SHADOW_MODE; try { diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 45a32547..7161271a 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -7,8 +7,15 @@ * 3. Classifying the ticket inline (priority, type, tags) * 4. Formatting the response for the source platform * 5. Persisting the AI response as a Message record - * 6. Enqueuing an ESCALATION job if confidence is too low, or if the pipeline - * suppressed an ungrounded draft + * 6. Enqueuing an ESCALATION job if confidence is too low, if the pipeline + * suppressed an ungrounded draft, or if the response never reached the + * reporter because platform delivery failed + * + * Delivery failure is escalated rather than swallowed because of the guard in + * step 1b (one response per ticket): once the BOT Message row exists, a retry or + * a manual re-enqueue is skipped, so an undelivered answer would otherwise leave + * the reporter permanently silent while the database claims they were answered. + * A human is the only remaining path, so the handler always pulls one in. * * The pipeline itself handles Pathfinder retrieval, Claude generation, * confidence scoring, platform-specific formatting, and the groundedness gate — @@ -156,6 +163,15 @@ export async function handleAiResponse( } console.log(`[AI Response] Confidence calibration: ${confidenceCalibration.toFixed(4)}`); + // Why the response never reached the reporter, when it didn't. Set by the + // post-back arm below and consumed by the escalation step: the one-response- + // per-ticket guard makes an undelivered answer unrecoverable by retry, so a + // human has to take the thread. + let deliveryFailure: string | null = null; + // Set when the ESCALATION enqueue itself failed after a delivery failure — + // the one case where the job must not report success (see the return below). + let escalationEnqueueError: string | null = null; + let pipelineResult; try { try { @@ -209,13 +225,28 @@ export async function handleAiResponse( }, }); - // Store the formatted response on the ticket for bots to pick up - await prisma.ticket.update({ - where: { id: ticket.id }, - data: { - suggestedResponse: pipelineResult.formatted.text, - }, - }); + // Store the formatted response on the ticket for bots to pick up. + // + // Non-fatal on purpose. The BOT Message row is already committed above, + // which arms the one-response-per-ticket guard — so if this write threw, + // the job would abort before post-back and every retry would be skipped + // by that guard, leaving the reporter permanently unanswered. Log it, + // remember it, and keep going so delivery still happens. + let suggestedResponseError: string | null = null; + try { + await prisma.ticket.update({ + where: { id: ticket.id }, + data: { + suggestedResponse: pipelineResult.formatted.text, + }, + }); + } catch (error) { + suggestedResponseError = error instanceof Error ? error.message : String(error); + console.error( + `[AI Response] Failed to store suggestedResponse for ticket ${ticketId}:`, + suggestedResponseError, + ); + } // 5b. Post the response back to the source platform — unconditionally. // @@ -268,17 +299,20 @@ export async function handleAiResponse( try { adapter = getAdapter(ticketSource); } catch (error) { + const message = error instanceof Error ? error.message : String(error); console.error( `[AI Response] Platform adapter misconfigured for ${ticket.source} on ticket ${ticketId}:`, - error instanceof Error ? error.message : String(error), + message, ); // Don't attempt postResponse — adapter init failed (permanent error) adapter = null; + deliveryFailure = `platform adapter misconfigured: ${message}`; } if (adapter) { + let externalCommentId: string | undefined; try { - const externalCommentId = await adapter.postResponse( + externalCommentId = await adapter.postResponse( { id: ticket.id, sourceId: ticket.sourceId, @@ -287,41 +321,68 @@ export async function handleAiResponse( }, pipelineResult.formatted, ); - if (externalCommentId) { - await prisma.message.update({ - where: { id: aiMessage.id }, - data: { externalCommentId }, - }); - } console.log( `[AI Response] Posted response to ${ticket.source} for ticket ${ticketId}`, ); } catch (error) { + const message = error instanceof Error ? error.message : String(error); console.error( `[AI Response] Failed to post response to ${ticket.source} for ticket ${ticketId}:`, - error instanceof Error ? error.message : String(error), + message, ); + deliveryFailure = message; + } + + // Recording the external comment ID is bookkeeping for an + // already-delivered response, so it gets its own try: a failure + // here must not be mistaken for a delivery failure. + if (externalCommentId) { + try { + await prisma.message.update({ + where: { id: aiMessage.id }, + data: { externalCommentId }, + }); + } catch (error) { + console.error( + `[AI Response] Failed to record externalCommentId for ticket ${ticketId}:`, + error instanceof Error ? error.message : String(error), + ); + } } } + } else if (suggestedResponseError) { + // No adapter for this source, so suggestedResponse WAS the delivery + // path — and that write failed. Nothing reached the reporter. + deliveryFailure = `no platform adapter for ${ticket.source} and suggestedResponse could not be stored: ${suggestedResponseError}`; } await context.reportProgress(85); - // 6. Enqueue ESCALATION when confidence is below threshold, or when the - // response was withheld — nothing reached the reporter in that case, so a - // human has to pick it up regardless of what the score says. - if (pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE || pipelineResult.suppressed) { + // 6. Enqueue ESCALATION when platform delivery failed, when the response + // was withheld, or when confidence is below threshold — in the first two + // cases nothing useful reached the reporter, so a human has to pick it up + // regardless of what the score says. Delivery failure wins the reason slot + // because it is the most actionable: the answer exists but is undelivered + // and, thanks to the one-response-per-ticket guard, undeliverable by retry. + const escalationReason = deliveryFailure + ? `AI response generated but not delivered to ${ticket.source} (${deliveryFailure}) — needs a human to answer the reporter` + : pipelineResult.suppressed + ? `AI response withheld (${pipelineResult.groundedness.reasons.join('; ')}) — needs a human answer` + : pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE + ? `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation` + : null; + + if (escalationReason) { try { await createJob(JobType.ESCALATION, { ticketId: ticket.id, - reason: pipelineResult.suppressed - ? `AI response withheld (${pipelineResult.groundedness.reasons.join('; ')}) — needs a human answer` - : `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation`, + reason: escalationReason, }); } catch (error) { + escalationEnqueueError = error instanceof Error ? error.message : String(error); console.error( `[AI Response] Failed to create escalation job for ticket ${ticketId}:`, - error instanceof Error ? error.message : String(error), + escalationEnqueueError, ); } } @@ -334,9 +395,30 @@ export async function handleAiResponse( console.log( `[AI Response] Ticket ${ticketId}: confidence=${pipelineResult.confidenceLevel} ` + `(${(pipelineResult.confidenceScore * 100).toFixed(0)}%), latency=${pipelineResult.latencyMs}ms` + - `${pipelineResult.suppressed ? ', ungrounded draft withheld' : ''}`, + `${pipelineResult.suppressed ? ', ungrounded draft withheld' : ''}` + + `${deliveryFailure ? `, delivery failed (${deliveryFailure})` : ''}`, ); + const escalated = + deliveryFailure !== null || + pipelineResult.suppressed || + pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE; + + // An undelivered answer with no escalation behind it is the one outcome that + // leaves the reporter silent and no human involved, and the guard blocks any + // retry from repairing it. Report failure so the attempt is recorded as failed + // and surfaces to an operator rather than being logged and forgotten. Other + // escalation-enqueue failures keep the historical success result: in those the + // response did reach the reporter. + if (deliveryFailure && escalationEnqueueError) { + return { + success: false, + error: + `Ticket ${ticketId}: AI response not delivered (${deliveryFailure}) and ` + + `escalation could not be enqueued (${escalationEnqueueError}) — needs manual attention`, + }; + } + return { success: true, data: { @@ -344,10 +426,11 @@ export async function handleAiResponse( confidenceLevel: pipelineResult.confidenceLevel, confidenceScore: pipelineResult.confidenceScore, latencyMs: pipelineResult.latencyMs, - escalated: - pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE || - pipelineResult.suppressed, + escalated, suppressed: pipelineResult.suppressed, + // Not `delivered` — shadow mode deliberately posts nothing, so only + // the failure is a fact worth reporting. + deliveryFailed: deliveryFailure !== null, }, }; } From f03bd5687c6139aaf276e3a1925348086ef0ab34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:30:36 -0400 Subject: [PATCH 64/83] fix(queue): answer the message that opened the ticket Outpost answers exactly one message per ticket -- the one that opened it (the invariant the handler already documents and enforces at step 1b). The question selection contradicted that: it scanned ticket.messages in REVERSE and took the LATEST type: 'USER' row. Replies are still persisted as USER messages, and that is correct -- they belong in the thread's history. The consequence was that a reporter who split a thought across two Discord messages in the seconds between ticket creation and the job dequeuing had the ticket's one and only answer aimed at the follow-up fragment ("btw I'm on the app router") instead of the question that opened the thread. One shot, spent on the wrong sentence. ticket.messages is loaded orderBy: { createdAt: 'asc' }, so the fix is a forward .find() for the first USER row. The ?? ticket.description ?? ticket.title fallback chain is unchanged. Judgement call -- conversation history stays FULL ------------------------------------------------ conversationHistory still carries every non-SYSTEM message, including any follow-up that landed after the opening message, even though the QUESTION is now the opening message. The two inputs answer different questions: `question` is what to respond to, `conversationHistory` is what the responder knows. A Discord follow-up is usually the same thought continued -- a stack trace, a version number, "on Next 15" -- and it is exactly the detail that makes the single allowed answer good, so truncating history would trade a targeting bug for a worse answer. Truncation would also require inventing a second policy for the non-USER rows after the opening, with no evidence behind it. Consecutive same-role turns are not a new condition: the generator already appends `question` after the history, so the base single-message case has always produced two user turns in a row. Call-Site Enumeration --------------------- - `latestUserMessage` (removed, function-local const): repo-wide grep over *.ts/*.tsx/*.md (excluding node_modules and dist) returns ZERO remaining references. Nothing outside the handler could see it; it never escaped the function body. Negative finding: no docs or comments named it either. - `openingUserMessage` (added, function-local const): 1 reference, the very next line (line 144). Not exported, not part of any type. No other call site can be affected. - `question` (unchanged name, changed VALUE): 1 consumer inside the handler -- `pipeline.generateSupportResponse(question, ...)` at line 175. Its contract is "a support question as a string"; still a string, still non-empty via the same fallback chain, so the assumption holds. The fallback ordering and the SUPPRESSED/groundedness paths downstream are value-agnostic and unaffected. - `pipeline.generateSupportResponse` (signature untouched): other callers are packages/outpost/ai/src/pipeline{,-groundedness}.test.ts (pass their own literal questions) and apps/web/src/app/api/qa/route.ts (passes the user's typed QA question, no ticket involved). Negative finding: none of them reads a Ticket's messages, so none inherits this behaviour change. - `conversationHistory` (unchanged shape and value): consumed by AIPipeline.generateSupportResponse -> generator.buildMessages (ai/src/generator.ts:222). Unchanged, so its assumptions hold by construction. - `ticket.messages` ordering assumption: the only producer is the `orderBy: { createdAt: 'asc' }` include at line 65 of this same file -- same function, no other loader feeds this variable. The step-1b already-answered guard also uses a forward `.find()` and is unaffected. Tests ----- New describe block "answers the message that opened the ticket" in queue/src/__tests__/ai-response.test.ts: the key case (opening USER + later USER -> generates against the opening, asserted on mockGenerateSupportResponse.mock.calls[0][0]), the companion assertion that the interim follow-up still arrives as conversationHistory, a leading-SYSTEM-row case, and the no-USER-message description fallback. The pre-existing "filters SYSTEM messages from conversation history" test encoded the bug in its expected question ('Follow up question') and now expects the opening ('Hello'); its history assertion is unchanged, which is what pins the judgement call above. Red-green verified: with the source reverted to the reverse().find() form, 3 tests fail (the two new targeting tests plus the corrected SYSTEM-filter test); restored, 968 tests pass. Gates: `npx vitest run --reporter=dot` in packages/outpost -> 60 files, 968 tests, all passing. `npx tsc --project queue/tsconfig.json --noEmit` -> clean (sibling packages built first so the workspace `dist` type entrypoints resolve). Prettier clean on both touched files. (cherry picked from commit 297e555ac62bd88b519a7fb90d7825b3965f43f4) --- .../queue/src/__tests__/ai-response.test.ts | 120 +++++++++++++++++- .../outpost/queue/src/handlers/ai-response.ts | 33 ++++- 2 files changed, 145 insertions(+), 8 deletions(-) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index c27b259e..a7937276 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -290,7 +290,7 @@ describe('handleAiResponse', () => { expect(result.data?.confidenceScore).toBe(0.92); expect(result.data?.escalated).toBe(false); - // Pipeline should have been called with the latest user message + // Pipeline should have been called with the message that opened the ticket expect(mockGenerateSupportResponse).toHaveBeenCalledWith( 'How do I use CopilotKit with Next.js?', expect.objectContaining({ @@ -1027,8 +1027,10 @@ describe('handleAiResponse', () => { await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + // Question is the OPENING message; the later USER turn is history, not + // the thing being answered. expect(mockGenerateSupportResponse).toHaveBeenCalledWith( - 'Follow up question', + 'Hello', expect.objectContaining({ conversationHistory: [ { role: 'user', content: 'Hello' }, @@ -1039,6 +1041,120 @@ describe('handleAiResponse', () => { ); }); + // ── The answered message is the OPENING message ─────────────────────── + // + // Outpost gets exactly one response per ticket, so which message that + // response addresses is the whole ballgame. Replies are persisted as USER + // messages by design, which is why "latest USER row" is not a safe proxy for + // "the question": a reporter who splits a thought across two Discord + // messages can land a second USER row before the job dequeues. + describe('answers the message that opened the ticket', () => { + /** Reporter follow-up landed before the job ran — the classic Discord split. */ + const splitThoughtTicket = { + ...sampleTicket, + messages: [ + { + id: 'msg-1', + type: 'USER', + content: 'How do I use CopilotKit with Next.js?', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:00Z'), + }, + { + id: 'msg-2', + type: 'USER', + content: 'btw I am on the app router', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:04Z'), + }, + ], + }; + + it('generates against the opening message, not a later follow-up', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(splitThoughtTicket); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(mockGenerateSupportResponse).toHaveBeenCalledTimes(1); + expect(mockGenerateSupportResponse.mock.calls[0]?.[0]).toBe( + 'How do I use CopilotKit with Next.js?', + ); + }); + + it('still passes the interim follow-up through as conversation context', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(splitThoughtTicket); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockGenerateSupportResponse.mock.calls[0]?.[1]).toMatchObject({ + conversationHistory: [ + { role: 'user', content: 'How do I use CopilotKit with Next.js?' }, + { role: 'user', content: 'btw I am on the app router' }, + ], + }); + }); + + it('skips leading non-USER rows to find the opening USER message', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + { + id: 'msg-0', + type: 'SYSTEM', + content: 'Ticket created from Discord thread', + isAiGenerated: false, + createdAt: new Date('2026-04-23T09:59:59Z'), + }, + { + id: 'msg-1', + type: 'USER', + content: 'Runtime returns 500 on /api/copilotkit', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:00Z'), + }, + { + id: 'msg-2', + type: 'USER', + content: 'here is the stack trace', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:06Z'), + }, + ], + }); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockGenerateSupportResponse.mock.calls[0]?.[0]).toBe( + 'Runtime returns 500 on /api/copilotkit', + ); + }); + + it('falls back to the description when the ticket has no USER message', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + { + id: 'msg-0', + type: 'SYSTEM', + content: 'Imported from Linear', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:00Z'), + }, + ], + }); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); + + expect(mockGenerateSupportResponse.mock.calls[0]?.[0]).toBe( + 'I want to add AI features to my Next.js app using CopilotKit.', + ); + }); + }); + // ── One response per ticket ─────────────────────────────────────────── // // The invariant: Outpost answers the message that opens a ticket and never diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 7161271a..021291fb 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -123,7 +123,11 @@ export async function handleAiResponse( }; } - // 2. Build conversation history from DB messages + // 2. Build conversation history from DB messages. + // + // Deliberately the FULL non-SYSTEM history, including any message that + // arrived after the one being answered. See the question selection below for + // why the two are allowed to disagree. const conversationHistory = ticket.messages .filter((m: { type: string }) => m.type !== 'SYSTEM') .map((m: { type: string; content: string }) => ({ @@ -131,11 +135,28 @@ export async function handleAiResponse( content: m.content, })); - // Determine the latest user message as the question - const latestUserMessage = [...ticket.messages] - .reverse() - .find((m: { type: string }) => m.type === 'USER'); - const question = latestUserMessage?.content ?? ticket.description ?? ticket.title; + // The question is the message that OPENED the ticket — the same message the + // one-response-per-ticket invariant above says we get to answer. + // + // `ticket.messages` is loaded `orderBy: { createdAt: 'asc' }`, so the FIRST + // USER row is the opening message. Scanning from the other end and taking + // the LATEST USER row was wrong: replies are still persisted as USER + // messages (correctly — they belong in the thread's history), so a reporter + // who splits a thought across two Discord messages in the seconds between + // ticket creation and this job running had the ticket's one and only answer + // aimed at the follow-up fragment instead of the question that opened it. + // One shot, spent on the wrong sentence. + // + // The interim follow-up deliberately STAYS in `conversationHistory`. Those + // two inputs answer different questions: `question` is what to respond to, + // `conversationHistory` is what the responder knows. A follow-up is usually + // the same thought continued — a stack trace, a version number, "on Next 15" + // — and it is exactly the detail that makes the single answer good, so + // dropping it would trade one bug for a worse answer. Suppressing it would + // also need a second policy for the non-USER rows after the opening, with no + // evidence behind it. + const openingUserMessage = ticket.messages.find((m: { type: string }) => m.type === 'USER'); + const question = openingUserMessage?.content ?? ticket.description ?? ticket.title; // Determine platform target for formatting const platform = payload.source ?? toPlatformTarget(ticket.source); From 57cbd281581b56e75f646b9d5043a0926b144e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:28:13 -0400 Subject: [PATCH 65/83] fix(queue): finish the skipped AI_RESPONSE job at 100% progress The one-response-per-ticket guard returned right after reportProgress(20), so a job that succeeded by deciding to do nothing persisted progress=20 on its Job row and read as hung to anything watching job progress. Walk the ladder to 100 before returning, like every other successful exit. Failure exits are left as-is on purpose: the Job row carries status FAILED next to the number, so a partial progress value is the honest reading there and 100 would falsely claim completion. (cherry picked from commit a65c2eaef1e088b24719d622468840415bc0a1e0) --- .../outpost/queue/src/__tests__/ai-response.test.ts | 13 +++++++++++++ packages/outpost/queue/src/handlers/ai-response.ts | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index a7937276..3320794c 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -1202,6 +1202,19 @@ describe('handleAiResponse', () => { expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); }); + it('drives progress to 100 so the skipped job is not left looking hung', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); + const ctx = makeContext(); + + await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, ctx); + + // The skip is a successful completion, so it must walk the ladder to + // 100 like the normal path. Returning after reportProgress(20) would + // persist a job stuck at 20% forever on the Job row. + expect(ctx.reportProgress).toHaveBeenCalledWith(100); + expect(ctx.reportProgress).toHaveBeenLastCalledWith(100); + }); + it('does not post anything to the platform for an already-answered ticket', async () => { mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 021291fb..9c3142a1 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -117,6 +117,13 @@ export async function handleAiResponse( `[AI Response] Ticket ${ticketId} already answered — skipping. ` + `Outpost posts one response per ticket; a human owns this thread now.`, ); + // Walk the ladder to 100 like every other successful exit. This job + // succeeded — it decided to do nothing — so anything reading job + // progress (dashboard, ops query) must see it finished, not parked at + // 20% looking hung. Failure exits deliberately leave progress where it + // stopped: the job row records status FAILED next to it, so a partial + // number is the honest reading there. + await context.reportProgress(100); return { success: true, data: { ticketId, skipped: true, reason: 'already_answered' }, From 7bd88b26f486f946a6fd293dc47c1c5274c96833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:28:32 -0400 Subject: [PATCH 66/83] test(queue): pin both halves of the one-response-per-ticket guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard's predicate is `m.type === 'BOT' && m.isAiGenerated`, but the tests only pinned the first half. Deleting `&& m.isAiGenerated` left the whole suite green, because no fixture carried a BOT row with isAiGenerated: false — the shape a human reply sent from the dashboard persists as. Two changes, tests only: - Spell out isAiGenerated on every message fixture. The DB column is non-nullable, so rows that omit it are a shape the handler never sees, and the omission let the guard be satisfied by `undefined`. - Add the BOT + isAiGenerated: false case: a teammate's reply goes out over the bot channel but is not Outpost's one answer, so the AI's single response must still be generated and posted. Mutation-verified: dropping `m.type === 'BOT'` fails the SYSTEM shadow-mode-log test; dropping `&& m.isAiGenerated` fails the new human-BOT-reply test (and the SYSTEM-history filter test). Each half is now independently pinned. (cherry picked from commit 35610d3fa08c9c37921508de21f36ecf02b765c4) --- .../queue/src/__tests__/ai-response.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index 3320794c..a717ff6b 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -135,9 +135,15 @@ const sampleTicket = { }, messages: [ { + // `isAiGenerated` is spelled out on every message fixture in this + // file: the DB column is non-nullable, so a row that omits it is a + // shape the handler never sees. Leaving it off let the + // one-response-per-ticket guard be satisfied by `undefined` instead + // of by a real `false`. id: 'msg-1', type: 'USER', content: 'How do I use CopilotKit with Next.js?', + isAiGenerated: false, createdAt: new Date('2026-04-23T10:00:00Z'), }, ], @@ -1001,24 +1007,31 @@ describe('handleAiResponse', () => { id: 'msg-1', type: 'USER', content: 'Hello', + isAiGenerated: false, createdAt: new Date('2026-04-23T10:00:00Z'), }, { + // A human reply sent from the dashboard: BOT row, but not + // the AI's answer, so it must not trip the guard and + // short-circuit this test before history is built. id: 'msg-2', type: 'BOT', content: 'Hi there!', + isAiGenerated: false, createdAt: new Date('2026-04-23T10:01:00Z'), }, { id: 'msg-3', type: 'SYSTEM', content: 'Ticket escalated', + isAiGenerated: false, createdAt: new Date('2026-04-23T10:02:00Z'), }, { id: 'msg-4', type: 'USER', content: 'Follow up question', + isAiGenerated: false, createdAt: new Date('2026-04-23T10:03:00Z'), }, ], @@ -1282,6 +1295,39 @@ describe('handleAiResponse', () => { expect(mockPostResponse).toHaveBeenCalled(); }); + it('does not treat a human BOT-channel reply as the ticket answer', async () => { + // A teammate answering from the dashboard persists as type 'BOT' + // with isAiGenerated: false — the outbound channel is the bot, the + // author is not. That is not Outpost's one response, so the AI's + // own single answer must still go out. + // + // Together with the SYSTEM case below this pins both halves of the + // guard's predicate independently: drop `m.type === 'BOT'` and the + // SYSTEM test goes red; drop `&& m.isAiGenerated` and this one does. + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-human', + type: 'BOT', + content: 'Hey, a maintainer here — can you share your version?', + isAiGenerated: false, + createdAt: new Date('2026-04-23T10:00:10Z'), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.data).not.toMatchObject({ skipped: true }); + expect(mockGenerateSupportResponse).toHaveBeenCalled(); + expect(mockPostResponse).toHaveBeenCalled(); + }); + it('does not treat a SYSTEM shadow-mode log as the ticket answer', async () => { // Shadow mode writes SYSTEM + isAiGenerated rows alongside the BOT // row. Only the BOT row means "the reporter has been answered", so a From aa8b98f141452aa2ed4570144e10e879971198c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:26:30 -0400 Subject: [PATCH 67/83] test: repoint team-member assertions off the reply path The one-response-per-ticket change made replies never enqueue an AI_RESPONSE for any sender, which quietly turned three "team member reply gets no AI response" tests tautological -- they would keep passing with team-member detection deleted outright. Team-member detection still decides aiJobEnqueued on the NEW-TICKET path, so that is where the assertion belongs. - packages/outpost/shared/.../platforms-inbound.test.ts: deleted 'skips AI_RESPONSE for team member reply'. The reply rule is already covered by two explicit tests, and the new-ticket path already has 'skips AI job when sender is a team member' plus the whole 'team member detection' block. A comment records why no team-member reply test lives there. - apps/discord-bot: deleted the reply-path test in message-create.test.ts and repointed it into thread-create.test.ts (discord's new-ticket path), which had no team-member coverage at all. - apps/slack-bot: repointed in place -- the same handler serves both paths, so the test moved from 'threaded replies' to 'new top-level messages'. Red-green verified with `return false` at the top of InboundHandler.isTeamMember: both repointed tests fail, and the three surviving shared-package team-member tests fail with them. Restored -> all green. No production code touched. (cherry picked from commit ad620a45f82648dff0f408a54b7daff186636d46) --- .../src/__tests__/message-create.test.ts | 25 ++------ .../src/__tests__/thread-create.test.ts | 24 ++++++++ apps/slack-bot/src/__tests__/message.test.ts | 61 +++++++++++-------- .../src/__tests__/platforms-inbound.test.ts | 23 +++---- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/apps/discord-bot/src/__tests__/message-create.test.ts b/apps/discord-bot/src/__tests__/message-create.test.ts index 39142554..25b4ba77 100644 --- a/apps/discord-bot/src/__tests__/message-create.test.ts +++ b/apps/discord-bot/src/__tests__/message-create.test.ts @@ -122,25 +122,12 @@ describe('handleMessageCreate', () => { expect(createJob).not.toHaveBeenCalled(); }); - it('does not enqueue AI response for team member messages', async () => { - // Set up as team member - vi.mocked(prisma.user.findFirst).mockResolvedValue({ - id: 'u-1', - email: 'team@copilotkit.ai', - } as ReturnType extends Promise ? T : never); - vi.mocked(prisma.teamMember.findUnique).mockResolvedValue({ - id: 'tm-1', - } as ReturnType extends Promise ? T : never); - - const message = makeMessage(); - await handleMessageCreate(message); - - // Should still save the message - expect(prisma.message.create).toHaveBeenCalled(); - - // Should NOT enqueue an AI response - expect(createJob).not.toHaveBeenCalled(); - }); + // No "does not enqueue AI response for team member messages" test here: + // every message this handler sees is a thread reply, and replies never + // enqueue for any sender, so it would pass with team-member detection + // removed entirely. The sender-dependent assertion now lives on the + // new-ticket path — see 'a team member opening a thread gets a ticket but + // no AI response' in thread-create.test.ts. it('reopens ticket when customer replies to a resolved ticket', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue({ diff --git a/apps/discord-bot/src/__tests__/thread-create.test.ts b/apps/discord-bot/src/__tests__/thread-create.test.ts index 1d2a81a3..bbdd7f27 100644 --- a/apps/discord-bot/src/__tests__/thread-create.test.ts +++ b/apps/discord-bot/src/__tests__/thread-create.test.ts @@ -128,6 +128,30 @@ describe('handleThreadCreate', () => { ); }); + // The new-ticket path is the one place where the sender still decides + // whether an AI job is enqueued: a community reporter's thread gets an + // answer (test above), a team member's does not. Replies never enqueue for + // anyone, so this assertion cannot live on the reply path. + it('creates a ticket but does not enqueue an AI response when a team member opens the thread', async () => { + vi.mocked(prisma.user.findFirst).mockResolvedValue({ + id: 'u-1', + email: 'team@copilotkit.ai', + } as ReturnType extends Promise ? T : never); + vi.mocked(prisma.teamMember.findUnique).mockResolvedValue({ + id: 'tm-1', + } as ReturnType extends Promise ? T : never); + + const thread = makeThread(); + await handleThreadCreate(thread, true); + + // The ticket and its first message are still recorded. + expect(prisma.ticket.create).toHaveBeenCalled(); + expect(prisma.message.create).toHaveBeenCalled(); + + // But the bot does not answer its own team. + expect(createJob).not.toHaveBeenCalled(); + }); + // The bot used to open every thread with "🎫 Ticket TKT-XXXXXXXX created…", // publishing an internal identifier into a public server and spending a bot // message on nothing the reporter can act on. The AI answer is the only diff --git a/apps/slack-bot/src/__tests__/message.test.ts b/apps/slack-bot/src/__tests__/message.test.ts index 2fefa492..bf8c0fc0 100644 --- a/apps/slack-bot/src/__tests__/message.test.ts +++ b/apps/slack-bot/src/__tests__/message.test.ts @@ -140,6 +140,36 @@ describe('registerMessageHandler', () => { expect(mockPostMessage).not.toHaveBeenCalled(); }); + // The new-ticket path is the one place where the sender still decides + // whether an AI job is enqueued: a community reporter's message gets an + // answer (test above), a team member's does not. Replies never enqueue + // for anyone, so this assertion cannot live on the reply path. + it('creates a ticket but does not enqueue an AI response when a team member opens the thread', async () => { + vi.mocked(prisma.user.findFirst).mockResolvedValue({ + id: 'u-1', + email: 'team@copilotkit.ai', + } as ReturnType extends Promise ? T : never); + vi.mocked(prisma.teamMember.findUnique).mockResolvedValue({ + id: 'tm-1', + } as ReturnType extends Promise ? T : never); + + await messageHandler({ + event: { + user: 'U_TEAM', + text: 'Heads up, deploying a fix shortly', + ts: '1234567890.123456', + channel: 'C_MONITORED', + }, + }); + + // The ticket and its first message are still recorded. + expect(prisma.ticket.create).toHaveBeenCalled(); + expect(prisma.message.create).toHaveBeenCalled(); + + // But the bot does not answer its own team. + expect(createJob).not.toHaveBeenCalled(); + }); + it('ignores messages in unmonitored channels', async () => { await messageHandler({ event: { @@ -212,32 +242,11 @@ describe('registerMessageHandler', () => { expect(createJob).not.toHaveBeenCalled(); }); - it('does not enqueue AI response for team member replies', async () => { - // Set up InboundHandler's isTeamMember via prisma mocks - vi.mocked(prisma.user.findFirst).mockResolvedValue({ - id: 'u-1', - email: 'team@copilotkit.ai', - } as ReturnType extends Promise ? T : never); - vi.mocked(prisma.teamMember.findUnique).mockResolvedValue({ - id: 'tm-1', - } as ReturnType extends Promise ? T : never); - - await messageHandler({ - event: { - user: 'U_TEAM', - text: 'Let me help you with that', - ts: '1234567891.000000', - thread_ts: '1234567890.123456', - channel: 'C_MONITORED', - }, - }); - - // Should still save the message - expect(prisma.message.create).toHaveBeenCalled(); - - // Should NOT enqueue AI response - expect(createJob).not.toHaveBeenCalled(); - }); + // No team-member variant of the test above: replies never enqueue for + // any sender, so asserting it for a team member would pass with + // team-member detection removed entirely. The sender-dependent + // assertion lives on the new-ticket path — see 'does not enqueue an AI + // response when a team member opens the thread'. it('reopens ticket when customer replies to a resolved ticket', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue({ diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index 88a8d660..c260ae21 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -337,22 +337,13 @@ describe('InboundHandler', () => { expect(createJob).not.toHaveBeenCalled(); }); - it('skips AI_RESPONSE for team member reply', async () => { - (prisma.user.findFirst as ReturnType).mockResolvedValue({ - id: 'user-db-1', - email: 'team@example.com', - }); - (prisma.teamMember.findUnique as ReturnType).mockResolvedValue({ - id: 'member-1', - }); - - const msg = makeInboundMessage({ isThreadStart: false }); - const result = await handler.handle(msg); - - expect(result.aiJobEnqueued).toBe(false); - expect(createJob).not.toHaveBeenCalled(); - }); - + // There is deliberately no "skips AI_RESPONSE for a team member reply" + // test here. Replies never enqueue for anyone (asserted above), so such + // a test would pass even if team-member detection were deleted. The + // sender-dependent assertion lives on the new-ticket path — see 'skips + // AI job when sender is a team member' and the 'team member detection' + // block. What a team member's reply DOES change is ticket status, which + // the next two tests cover. it('transitions WAITING_ON_TEAM to WAITING_ON_CUSTOMER when team member replies', async () => { (prisma.ticket.findFirst as ReturnType).mockResolvedValue({ ...existingTicket, From ee502a0c1a4e1cd778b897e7d64ffb52b3d9422e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:49:32 -0400 Subject: [PATCH 68/83] docs: correct the one-response-per-ticket comments to match the code The prose around the one-answer invariant made claims the code does not support, and framed the enforcement backwards. Corrections: - inbound.ts `handle()` / module header: path 1 enqueues AI_RESPONSE only when the sender is not a team member, and it is the only path here that enqueues at all. Team status no longer gates the reply path in any way. - ai-response.ts header: the step list omitted step 1b (already-answered gate), step 5b (platform post-back), and the SHADOW_MODE branch that replaces post-back with a SYSTEM message. - ai-response.ts gate comment: "Five separate code paths could enqueue AI_RESPONSE" was both miscounted and wrong about which. Three enqueue sites exist -- InboundHandler.handleNewTicket (Discord, Slack and Teams all funnel through it), handleShadowThreadCreate in the Discord shadow-mode path, and the Postmark new-email branch. - The gate is a RE-ANSWER guard, not the enforcement point, and the comments now say so everywhere they mention it. A ticket freshly minted around a mid-thread reply carries no prior AI response and passes the gate untouched; likewise a reply on a ticket Outpost never answered. The enqueue-site refusals are what actually hold one-answer-per-ticket, so calling them a cost optimisation or "the cheap arm" was backwards. - source-id.ts: a writer/reader key mismatch no longer means "every reply gets its own AI answer" -- an unmatched reply is filed as an untracked ticket with no answer. The real damage is the lost reply and the duplicate stub. - shadow-mode.ts: the enqueue comment said the worker calls logShadowResponse; it writes the shadow row inline and never calls it. The logShadowResponse docstring said NOTE-type; the row is SYSTEM. Also trimmed the "this used to..." narrative, which was repeated in four places, down to the two sites where it genuinely stops the bug being reintroduced. Comments only -- no behavior change, so no test accompanies it. Verified with `npx turbo typecheck` and `npx turbo test` (all green). (cherry picked from commit 69b4c941a8d8ec18e4b70f65c6b214a3c66bf14d) --- apps/discord-bot/src/lib/shadow-mode.ts | 8 ++- apps/github-app/src/webhooks/issue-comment.ts | 8 +-- .../src/app/api/webhooks/postmark/route.ts | 6 +- .../outpost/queue/src/handlers/ai-response.ts | 55 +++++++++++-------- .../outpost/shared/src/platforms/inbound.ts | 39 +++++++------ .../outpost/shared/src/platforms/source-id.ts | 7 ++- 6 files changed, 72 insertions(+), 51 deletions(-) diff --git a/apps/discord-bot/src/lib/shadow-mode.ts b/apps/discord-bot/src/lib/shadow-mode.ts index 5a0763e0..f387a79d 100644 --- a/apps/discord-bot/src/lib/shadow-mode.ts +++ b/apps/discord-bot/src/lib/shadow-mode.ts @@ -27,7 +27,7 @@ export interface ShadowResponse { /** * Log a shadow response for later quality comparison. - * Stored as a NOTE-type message on the ticket with metadata in attachments. + * Stored as a SYSTEM-type message on the ticket with metadata in attachments. */ export async function logShadowResponse(response: ShadowResponse): Promise { await prisma.message.create({ @@ -91,8 +91,10 @@ export async function handleShadowThreadCreate( }); } - // Enqueue AI response — the worker should check shadow mode - // and call logShadowResponse instead of posting to Discord + // Enqueue the one AI response this ticket gets. The handler reads + // SHADOW_MODE itself and, when it is set, logs the generated response as + // a SYSTEM message on the ticket instead of posting it to Discord (it + // writes that row inline — it does not call logShadowResponse below). await createJob(JobType.AI_RESPONSE, { ticketId: ticket.id, threadId: thread.id, diff --git a/apps/github-app/src/webhooks/issue-comment.ts b/apps/github-app/src/webhooks/issue-comment.ts index 79aad339..607c54a0 100644 --- a/apps/github-app/src/webhooks/issue-comment.ts +++ b/apps/github-app/src/webhooks/issue-comment.ts @@ -78,10 +78,10 @@ export async function handleIssueComment( } } else { // No AI response on comments — Outpost answers the issue body once and - // then stays out of the thread, whoever comments next. This previously - // enqueued an AI_RESPONSE for every non-team commenter, so the bot kept - // replying to follow-ups on issues a human had already picked up. - // The AI_RESPONSE handler enforces the same invariant server-side. + // then stays out of the thread, whoever comments next. Not enqueuing + // here is what enforces that; the AI_RESPONSE handler's + // already-answered gate only backstops re-answering a ticket that + // already holds an AI response. // Reopen a dormant ticket so a human sees the follow-up. The status // set lives in @copilotkit/outpost/shared so this path, the shared diff --git a/apps/web/src/app/api/webhooks/postmark/route.ts b/apps/web/src/app/api/webhooks/postmark/route.ts index 08aa3343..37d9d99f 100644 --- a/apps/web/src/app/api/webhooks/postmark/route.ts +++ b/apps/web/src/app/api/webhooks/postmark/route.ts @@ -94,8 +94,10 @@ export async function POST(request: Request) { } // No AI response on a reply — Outpost answers the opening email - // once and a human handles the rest of the thread. The - // AI_RESPONSE handler enforces the same invariant server-side. + // once and a human handles the rest of the thread. Not enqueuing + // here is what enforces that; the AI_RESPONSE handler's + // already-answered gate only backstops re-answering a ticket that + // already holds an AI response. return NextResponse.json({ status: 'message_appended', ticketId: existingTicket.displayId }); } diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 9c3142a1..04469ec3 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -2,14 +2,20 @@ * AI_RESPONSE job handler. * * The most critical handler in Outpost. Processes AI_RESPONSE jobs by: - * 1. Loading the ticket and its messages from the database - * 2. Running the AI pipeline to generate a support response - * 3. Classifying the ticket inline (priority, type, tags) - * 4. Formatting the response for the source platform - * 5. Persisting the AI response as a Message record - * 6. Enqueuing an ESCALATION job if confidence is too low, if the pipeline - * suppressed an ungrounded draft, or if the response never reached the - * reporter because platform delivery failed + * 1. Loading the ticket and its messages from the database + * 1b. Finishing immediately, as a success, if the ticket already holds an AI + * response — one response per ticket (see the gate below) + * 2. Running the AI pipeline to generate a support response + * 3. Classifying the ticket inline (priority, type, tags) + * 4. Formatting the response for the source platform + * 5. Persisting the AI response as a Message record, and the formatted text + * on the ticket as suggestedResponse + * 5b. Posting the response back to the source platform through its adapter — + * except under SHADOW_MODE=true, where the response is instead logged as a + * SYSTEM message on the ticket and nothing is posted anywhere + * 6. Enqueuing an ESCALATION job if confidence is too low, if the pipeline + * suppressed an ungrounded draft, or if the response never reached the + * reporter because platform delivery failed * * Delivery failure is escalated rather than swallowed because of the guard in * step 1b (one response per ticket): once the BOT Message row exists, a retry or @@ -82,7 +88,7 @@ export async function handleAiResponse( await context.reportProgress(20); - // 1b. ONE RESPONSE PER TICKET — hard invariant, enforced here. + // 1b. ONE RESPONSE PER TICKET — RE-ANSWER guard. // // Outpost answers exactly one message per ticket: the one that opened it. // Every later message in that thread gets no AI reply, no matter who sent @@ -90,21 +96,24 @@ export async function handleAiResponse( // a first line of defence and a human owns the thread from the moment the // first response lands. // - // The gate lives in the handler rather than at the enqueue sites on - // purpose. Five separate code paths could enqueue AI_RESPONSE (Discord, - // Slack, Teams, the GitHub comment webhook, the Postmark reply webhook) and - // each one previously decided for itself whether a reply warranted an - // answer. Those enqueues are gone, but a single new caller added later - // would silently reintroduce the follow-up spam this closes. Checking the - // ticket's own history catches every re-answer of a ticket we already - // answered. + // What this gate does and does not do, because the distinction matters: // - // It is NOT a total gate, so do not lean on it as one. It can only see - // messages on the ticket, so it cannot tell a first answer from a first - // answer to the wrong message: a ticket freshly minted around a mid-thread - // message has no prior AI response and would sail through here. That case - // (an orphaned reply, no ticket found for the thread) is refused at the - // enqueue site in InboundHandler.handleReply — see the comment there. + // The invariant is enforced at the enqueue sites, not here. Three of them + // exist — InboundHandler.handleNewTicket (Discord, Slack and Teams all + // funnel through it), handleShadowThreadCreate in the Discord bot's + // shadow-mode path, and the Postmark webhook's new-email branch — and every + // one enqueues only for a message that opens a ticket. Their refusal to + // enqueue for anything else is what holds the rule. + // + // This gate catches the second answer to a ticket that already has one: a + // retried job, a manual re-enqueue, or a caller added later that does not + // respect the rule. It CANNOT stand in for those refusals, so do not lean + // on it as if it could. It only sees messages on the ticket, so it cannot + // tell a first answer from a first answer to the wrong message: a ticket + // freshly minted around a mid-thread reply carries no prior AI response and + // sails straight through here. That case (an orphaned reply, no ticket found + // for the thread) is refused where the ticket is created — see + // InboundHandler.handleReply. // // Success, not failure: the job did what it should — nothing. Returning an // error would put it through the retry ladder for a decision that will diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 554d7bfd..20eb5951 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -3,10 +3,13 @@ * a raw platform event into an InboundMessage. * * Handles: - * 1. New tickets (isThreadStart=true): create Ticket + first Message + enqueue AI_RESPONSE - * 2. Replies (isThreadStart=false): find existing ticket, create Message, reopen if needed. - * Never enqueues AI_RESPONSE — Outpost answers once per ticket, on the opening - * message only, and a human owns the thread after that. + * 1. New tickets (isThreadStart=true): create Ticket + first Message, and enqueue + * AI_RESPONSE unless the sender is a team member. This is the only path here + * that ever enqueues. + * 2. Replies (isThreadStart=false): find existing ticket, create Message, reopen if + * needed. Never enqueues AI_RESPONSE, whoever sent the reply — Outpost answers + * once per ticket, on the opening message only, and a human owns the thread + * after that. * 3. Orphaned replies (isThreadStart=false with no matching ticket): create the * Ticket + Message so the customer's words are never dropped, but do NOT * enqueue AI_RESPONSE — we never saw the message that opened the conversation. @@ -257,10 +260,13 @@ export class InboundHandler { // this reply is not "the message that opened the ticket" in the // product sense even though it is the ticket's first message. Outpost // answers exactly one message per ticket — the opening one — and this - // is not it. Answering here is how a follow-up ("any update?", or a - // community member's reply to someone else) used to get an AI reply - // in a thread Outpost was never part of; that routed around the - // one-answer-per-ticket rule entirely. + // is not it. + // + // Refusing here is the only thing that stops it. The ticket we are + // about to create carries no prior AI response, so the + // already-answered gate in the AI_RESPONSE handler would wave it + // straight through and answer a mid-thread "any update?" in a + // conversation Outpost was never part of. return this.handleNewTicket({ ...message, isThreadStart: true }, { answer: false }); } @@ -281,15 +287,16 @@ export class InboundHandler { // // Outpost answers the message that opens a ticket and nothing after it. // Replies only move ticket state; the thread belongs to a human from - // the first response onward. This used to enqueue an AI_RESPONSE for - // every non-team sender, which meant the bot chimed in on follow-up - // questions between community members and even summarised a human's - // answer back at them. + // the first response onward. Enqueuing here is what made the bot chime + // in on follow-up questions between community members and summarise a + // human's answer back at them. // - // The invariant is also enforced in the AI_RESPONSE handler - // (packages/outpost/queue/src/handlers/ai-response.ts) against the - // ticket's own message history. Not enqueuing here is the cheap arm — - // it avoids paying for a job that would be dropped on arrival. + // This refusal is what enforces the invariant. The already-answered + // gate in the AI_RESPONSE handler + // (packages/outpost/queue/src/handlers/ai-response.ts) is a backstop + // against re-answering a ticket that already holds an AI response, not + // a substitute: a reply on a ticket Outpost never answered — one opened + // by a team member, say — would pass that gate untouched. const isTeam = await this.isTeamMember(message.platformUserId, message.source); if (isTeam) { diff --git a/packages/outpost/shared/src/platforms/source-id.ts b/packages/outpost/shared/src/platforms/source-id.ts index c0c10dc2..0eb5f8e7 100644 --- a/packages/outpost/shared/src/platforms/source-id.ts +++ b/packages/outpost/shared/src/platforms/source-id.ts @@ -5,9 +5,10 @@ * `Ticket.sourceId` is the platform-thread key Outpost uses to decide whether * an inbound message opens a new ticket or belongs to an existing one. It is * written once (on ticket create) and read on every reply. Those two sites MUST - * derive the key identically or every reply looks like a brand-new ticket and - * gets its own AI answer — the exact bug this module exists to make - * unrepresentable. Do not inline `${channelId}:${threadId}` anywhere; call + * derive the key identically: if they disagree, every reply misses the ticket it + * belongs to and is filed as a separate untracked one, so the real ticket loses + * the reply (no message appended, no reopen) and the dashboard fills with + * duplicate stubs. Do not inline `${channelId}:${threadId}` anywhere; call * `buildTicketSourceId`. */ From aa95e94e8eb1efc94d94155e9264b07197944215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:29:20 -0400 Subject: [PATCH 69/83] fix(teams): gate replies by monitored channel Call sites audited: - handleMessage dispatch: apps/teams-bot/src/index.ts -> handlers/message.ts - TeamsAdapter.parseInboundEvent: handlers/message.ts; teams-adapter.test.ts; shared platforms-adapters.test.ts - orphan reply path: InboundHandler.handleReply -> handleNewTicket(answer=false) -> Teams isNewTicket acknowledgment branch - policy tests: message.test.ts; teams-adapter.test.ts; inbound-handler.test.ts --- apps/teams-bot/src/__tests__/message.test.ts | 15 +++++++++++++++ apps/teams-bot/src/handlers/message.ts | 17 ++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/apps/teams-bot/src/__tests__/message.test.ts b/apps/teams-bot/src/__tests__/message.test.ts index 7f17733f..8d637dbd 100644 --- a/apps/teams-bot/src/__tests__/message.test.ts +++ b/apps/teams-bot/src/__tests__/message.test.ts @@ -174,6 +174,21 @@ describe('handleMessage', () => { expect(prisma.ticket.create).not.toHaveBeenCalled(); }); + it('ignores replies in unmonitored channels before orphan fallback', async () => { + const context = makeContext({ + replyToId: 'missing-parent-id', + channelData: { teamsChannelId: 'unmonitored-channel' }, + }); + + await handleMessage(context); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + expect(prisma.ticket.create).not.toHaveBeenCalled(); + expect(prisma.message.create).not.toHaveBeenCalled(); + expect(createJob).not.toHaveBeenCalled(); + expect(context.sendActivity).not.toHaveBeenCalled(); + }); + it('appends follow-up messages to existing tickets', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue( TICKET as ReturnType extends Promise ? T : never, diff --git a/apps/teams-bot/src/handlers/message.ts b/apps/teams-bot/src/handlers/message.ts index 02713fc0..2103d739 100644 --- a/apps/teams-bot/src/handlers/message.ts +++ b/apps/teams-bot/src/handlers/message.ts @@ -50,16 +50,15 @@ export async function handleMessage(context: TurnContext): Promise { if (!message || !message.content) return; try { - // Channel monitoring filter: if monitoredChannelIds is configured, - // only process messages from those channels. If empty, monitor all. - if (message.isThreadStart) { - const channelId = message.channelId; - const isMonitored = - config.monitoredChannelIds.length === 0 || - (channelId !== undefined && config.monitoredChannelIds.includes(channelId)); + // Channel monitoring filter applies to both thread starts and replies. + // If the list is empty, monitor all channels (including 1:1 chats); + // otherwise the activity must carry an explicitly monitored channel ID. + const channelId = message.channelId; + const isMonitored = + config.monitoredChannelIds.length === 0 || + (channelId !== undefined && config.monitoredChannelIds.includes(channelId)); - if (!isMonitored) return; - } + if (!isMonitored) return; // Delegate to the shared inbound handler const result = await inboundHandler.handle(message); From 9180591c84166f561ae569193ad23f64de189100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:53:30 -0400 Subject: [PATCH 70/83] fix(inbound): keep the bot silent on an orphaned reply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #170 caught a regression this branch introduced. The orphaned-reply fallback creates a ticket with `{ answer: false }` so no AI response is enqueued, but it routes through `handleNewTicket`, which returns `isNewTicket: true` unconditionally. The Teams handler branches on exactly that flag to write a conversationReference and post an acknowledgement card. So a mid-thread "thanks, that worked!" -- Teams sets isThreadStart from the absence of replyToId, and no ticket matches -- made the bot post a card into a conversation it was never part of. That is the unwanted-chatter class this branch exists to remove, reintroduced on the platform most reachable for the orphan path. `InboundResult` now carries `isOrphanedReply`, set true only on that fallback, and `handleNewTicket` takes `{ answer, orphanedReply }` as two independent decisions rather than inferring one from the other. The Teams handler gates both the card and the additionalInfo write on it. `isNewTicket` stays true: a ticket really was created, and other consumers may reasonably care. The existing handleMessage orphan test asserted the bug -- it expected sendActivity to have been called once -- so it would have kept this green forever. It now asserts the ticket and message are still persisted while createJob, sendActivity and ticket.update are not called. Verified red twice: reverting the gate fails it, and gating only the card while leaving the additionalInfo write ungated also fails it, so each half is independently covered. Call-site enumeration for isOrphanedReply -- 1 consumer changed, 7 cleared: - apps/teams-bot/src/handlers/message.ts -- CHANGED. The only consumer with a reporter-visible side effect on the new-ticket branch. - apps/slack-bot/src/events/message.ts -- cleared: pre-filters replies whose thread has no ticket, so it cannot reach the fallback. - apps/discord-bot/src/events/message-create.ts -- cleared: same pre-filter. - apps/github-app/src/webhooks/issue-comment.ts -- cleared: returns when no ticket matches, and never uses InboundHandler. - apps/github-app/src/webhooks/issues-opened.ts and discussion-created.ts -- cleared: isThreadStart is always true, and neither has a reporter-visible side effect on that branch. - apps/discord-bot/src/events/thread-create.ts -- cleared: no ack post since this branch removed it. - apps/web -- cleared: never uses InboundHandler; Postmark has its own path and already implements the orphan rule locally. Two further review findings, same shape as the above: Route the last three inlined `Ticket.sourceId` sites through `buildTicketSourceId` -- shadow-mode.ts on the write side, discord-bot and teams-bot lib/tickets.ts on the read side. Identity for both platforms today, so this fixes no live bug; it closes the drift surface the helper exists for, which already produced one write/read mismatch on this branch. Readers return early on an unaddressable key instead of querying `sourceId: null`, which would match unrelated keyless rows. Adds tickets.test.ts for discord-bot, which had none, and stops shadow-mode.test.ts stubbing the whole shared module -- the helper was mocked away, so no test there could have caught a drift. Correct comment prose that overstated what the code does. The literal — escape was in four files, not the two previously fixed -- github-app's issues-opened and discussion-created carried it too, plus a mangled ellipsis. The orphaned-reply rationale in inbound.ts read as a universal principle, but Discord, GitHub and Slack all pre-filter before reaching it and Postmark implements it locally, so Teams is the only caller that arrives there; the comment now says so. The Teams ack card's divergence from Discord and Slack, which post nothing, is recorded as deliberate rather than left to look like an oversight. Not addressed here, both belonging to the email-threading follow-up: postmark/route.ts inlines the EMAIL key as body.MessageID in three places, and a plain reply with no plus-address is still treated as new and answered, because sourceId is the per-email MessageID and there is no In-Reply-To or References fallback. --- .../src/__tests__/shadow-mode.test.ts | 58 ++++++++++- .../discord-bot/src/__tests__/tickets.test.ts | 96 +++++++++++++++++++ apps/discord-bot/src/events/thread-create.ts | 2 +- apps/discord-bot/src/lib/shadow-mode.ts | 11 ++- apps/discord-bot/src/lib/tickets.ts | 16 +++- .../src/__tests__/discussion-created.test.ts | 1 + .../src/__tests__/issues-opened.test.ts | 1 + .../src/webhooks/discussion-created.ts | 2 +- apps/github-app/src/webhooks/issues-opened.ts | 4 +- apps/slack-bot/src/events/message.ts | 2 +- apps/teams-bot/src/__tests__/message.test.ts | 27 ++++++ apps/teams-bot/src/__tests__/tickets.test.ts | 26 +++++ apps/teams-bot/src/handlers/message.ts | 29 +++++- apps/teams-bot/src/lib/tickets.ts | 16 +++- .../src/__tests__/platforms-inbound.test.ts | 10 +- .../outpost/shared/src/platforms/inbound.ts | 46 ++++++++- .../outpost/shared/src/platforms/types.ts | 13 +++ 17 files changed, 342 insertions(+), 18 deletions(-) create mode 100644 apps/discord-bot/src/__tests__/tickets.test.ts diff --git a/apps/discord-bot/src/__tests__/shadow-mode.test.ts b/apps/discord-bot/src/__tests__/shadow-mode.test.ts index d3573d2b..6b873ee7 100644 --- a/apps/discord-bot/src/__tests__/shadow-mode.test.ts +++ b/apps/discord-bot/src/__tests__/shadow-mode.test.ts @@ -6,9 +6,17 @@ import { mockPrisma, mockQueue } from './helpers/mocks.js'; vi.mock('@copilotkit/outpost/db', () => mockPrisma()); vi.mock('@copilotkit/outpost/queue', () => mockQueue()); -vi.mock('@copilotkit/outpost/shared', () => ({ - truncate: vi.fn((str: string, _len: number) => str), -})); +// truncate is stubbed to a pass-through so assertions can compare exact +// strings, but buildTicketSourceId/TicketSource stay REAL: the point of the +// sourceId assertions below is that shadow mode derives the key with the shared +// helper, which a stub would hide. +vi.mock('@copilotkit/outpost/shared', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + truncate: vi.fn((str: string, _len: number) => str), + }; +}); import { isShadowMode, @@ -18,6 +26,7 @@ import { } from '../lib/shadow-mode.js'; import { prisma } from '@copilotkit/outpost/db'; import { createJob, JobType } from '@copilotkit/outpost/queue'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; function makeThread(overrides: Record = {}) { return { @@ -130,6 +139,49 @@ describe('shadow-mode', () => { }); }); + // The writer must derive sourceId with buildTicketSourceId, not inline + // thread.id: findTicketByThreadId reads through that helper, so an + // inlined write silently desynchronizes the moment the derivation + // changes (exactly the null-vs-'' bug this helper was introduced for). + it('derives sourceId with buildTicketSourceId, not an inlined thread.id', async () => { + const thread = makeThread({ id: 'thread-999' }); + await handleShadowThreadCreate( + thread, + 'TKT-0001', + 'My question', + 'TestUser#1234', + 'user-456', + ); + + const { data } = vi.mocked(prisma.ticket.create).mock.calls[0]![0] as { + data: { sourceId: string | null }; + }; + expect(data.sourceId).toBe( + buildTicketSourceId(TicketSource.DISCORD, 'thread-999'), + ); + expect(data.sourceId).toBe('thread-999'); + }); + + it('stores a null sourceId for an unaddressable thread', async () => { + // No thread key means no reply can ever find this ticket. We still + // create it (never drop the report) but must store the helper's null + // rather than an empty-string placeholder a reader would search for. + const thread = makeThread({ id: '' }); + const result = await handleShadowThreadCreate( + thread, + 'TKT-0001', + 'My question', + 'TestUser#1234', + 'user-456', + ); + + expect(result).toBe('ticket-internal-id'); + const { data } = vi.mocked(prisma.ticket.create).mock.calls[0]![0] as { + data: { sourceId: string | null }; + }; + expect(data.sourceId).toBeNull(); + }); + it('creates a message record for the content', async () => { const thread = makeThread(); await handleShadowThreadCreate( diff --git a/apps/discord-bot/src/__tests__/tickets.test.ts b/apps/discord-bot/src/__tests__/tickets.test.ts new file mode 100644 index 00000000..eaacc0fb --- /dev/null +++ b/apps/discord-bot/src/__tests__/tickets.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { mockPrisma } from './helpers/mocks.js'; + +vi.mock('@copilotkit/outpost/db', () => mockPrisma()); + +import { findTicketByThreadId, isTeamMember } from '../lib/tickets.js'; +import { prisma } from '@copilotkit/outpost/db'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; + +describe('findTicketByThreadId', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('queries by the sourceId buildTicketSourceId derives', async () => { + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + await findTicketByThreadId('thread-123'); + + expect(prisma.ticket.findFirst).toHaveBeenCalledWith({ + where: { + source: 'DISCORD', + sourceId: buildTicketSourceId(TicketSource.DISCORD, 'thread-123'), + }, + }); + // Pinned literal too, so a derivation change has to be a deliberate act + // in both writer and reader rather than a silently-agreeing tautology. + expect(prisma.ticket.findFirst).toHaveBeenCalledWith({ + where: { source: 'DISCORD', sourceId: 'thread-123' }, + }); + }); + + it('returns the ticket when found', async () => { + const ticket = { id: 'ticket-1', source: 'DISCORD', sourceId: 'thread-123' }; + vi.mocked(prisma.ticket.findFirst).mockResolvedValue( + ticket as ReturnType extends Promise ? T : never, + ); + + expect(await findTicketByThreadId('thread-123')).toEqual(ticket); + }); + + it('returns null when no ticket found', async () => { + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + expect(await findTicketByThreadId('nonexistent')).toBeNull(); + }); + + it('skips the query entirely for an unaddressable thread', async () => { + // buildTicketSourceId yields no key for an empty thread ID. Falling + // through to `sourceId: null` would match any keyless row and hand back + // an unrelated ticket, so the lookup must not run at all. + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + expect(await findTicketByThreadId('')).toBeNull(); + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); +}); + +describe('isTeamMember', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns true when the user maps to a TeamMember', async () => { + vi.mocked(prisma.user.findFirst).mockResolvedValue({ + id: 'u-1', + email: 'team@copilotkit.ai', + } as ReturnType extends Promise ? T : never); + vi.mocked(prisma.teamMember.findUnique).mockResolvedValue({ + id: 'tm-1', + } as ReturnType extends Promise ? T : never); + + expect(await isTeamMember('discord-user-1')).toBe(true); + expect(prisma.user.findFirst).toHaveBeenCalledWith({ + where: { externalId: 'discord-user-1', source: 'DISCORD' }, + }); + }); + + it('returns false when no User row exists', async () => { + vi.mocked(prisma.user.findFirst).mockResolvedValue(null); + + expect(await isTeamMember('unknown')).toBe(false); + }); + + it('returns false when the User has no email', async () => { + // `email` is non-nullable in the schema, so "no email" surfaces as the + // empty string — which the `!user?.email` guard must still reject. + vi.mocked(prisma.user.findFirst).mockResolvedValue({ + id: 'u-1', + email: '', + } as ReturnType extends Promise ? T : never); + + expect(await isTeamMember('no-email')).toBe(false); + expect(prisma.teamMember.findUnique).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/discord-bot/src/events/thread-create.ts b/apps/discord-bot/src/events/thread-create.ts index 6006f032..3eccb568 100644 --- a/apps/discord-bot/src/events/thread-create.ts +++ b/apps/discord-bot/src/events/thread-create.ts @@ -85,7 +85,7 @@ export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: bo // "\uD83C\uDFAB Ticket TKT-XXXXXXXX created..." in the thread, which leaked an // internal identifier to the public server and spent a bot message // saying nothing the reporter can act on. displayId is for the dashboard - // and team slash commands only \u2014 never for reporter-facing copy. + // and team slash commands only — never for reporter-facing copy. // The AI response itself is the only message the reporter needs. console.log(`[Discord Bot] Created ticket ${result.displayId} for thread ${thread.id}`); diff --git a/apps/discord-bot/src/lib/shadow-mode.ts b/apps/discord-bot/src/lib/shadow-mode.ts index f387a79d..06495249 100644 --- a/apps/discord-bot/src/lib/shadow-mode.ts +++ b/apps/discord-bot/src/lib/shadow-mode.ts @@ -1,6 +1,6 @@ import { prisma } from '@copilotkit/outpost/db'; import { createJob, JobType } from '@copilotkit/outpost/queue'; -import { truncate } from '@copilotkit/outpost/shared'; +import { TicketSource, buildTicketSourceId, truncate } from '@copilotkit/outpost/shared'; import type { ThreadChannel, Message } from 'discord.js'; /** @@ -64,6 +64,13 @@ export async function handleShadowThreadCreate( authorId: string, ): Promise { try { + // Build the lookup key through the SAME helper findTicketByThreadId + // reads with, so the stored key and the searched-for key cannot drift + // apart. null means "this thread is not addressable" (no thread ID) — + // the ticket is still created so the report is not dropped, but no later + // reply will match it. + const sourceId = buildTicketSourceId(TicketSource.DISCORD, thread.id); + const ticket = await prisma.ticket.create({ data: { displayId, @@ -73,7 +80,7 @@ export async function handleShadowThreadCreate( priority: 'MEDIUM', type: 'QUESTION', source: 'DISCORD', - sourceId: thread.id, + sourceId, sourceUrl: thread.url, channel: thread.parentId ?? undefined, }, diff --git a/apps/discord-bot/src/lib/tickets.ts b/apps/discord-bot/src/lib/tickets.ts index 0b8128ff..cd19dcf6 100644 --- a/apps/discord-bot/src/lib/tickets.ts +++ b/apps/discord-bot/src/lib/tickets.ts @@ -1,13 +1,27 @@ import { prisma } from '@copilotkit/outpost/db'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; /** * Find a ticket by its Discord thread ID (stored as sourceId with source=DISCORD). + * + * The key is built by buildTicketSourceId — the same helper InboundHandler and + * shadow mode store tickets with — so this lookup can never search for a + * spelling nothing was written under. For Discord that is the thread ID + * verbatim today, but routing through the helper is what keeps it that way: the + * next change to the derivation moves writer and reader together. + * + * A null key (no thread ID) means no ticket can carry it, so there is nothing + * to query — returning early also stops a `sourceId: null` filter from matching + * an unrelated keyless row. */ export async function findTicketByThreadId(threadId: string) { + const sourceId = buildTicketSourceId(TicketSource.DISCORD, threadId); + if (sourceId === null) return null; + return prisma.ticket.findFirst({ where: { source: 'DISCORD', - sourceId: threadId, + sourceId, }, }); } diff --git a/apps/github-app/src/__tests__/discussion-created.test.ts b/apps/github-app/src/__tests__/discussion-created.test.ts index 81f21118..e830e156 100644 --- a/apps/github-app/src/__tests__/discussion-created.test.ts +++ b/apps/github-app/src/__tests__/discussion-created.test.ts @@ -8,6 +8,7 @@ const mockHandleResult = { ticketId: 'ticket-disc-id', displayId: 'TKT-DS01', isNewTicket: true, + isOrphanedReply: false, aiJobEnqueued: true, messageId: 'message-disc-id', }; diff --git a/apps/github-app/src/__tests__/issues-opened.test.ts b/apps/github-app/src/__tests__/issues-opened.test.ts index 4506ee3b..84e76d31 100644 --- a/apps/github-app/src/__tests__/issues-opened.test.ts +++ b/apps/github-app/src/__tests__/issues-opened.test.ts @@ -9,6 +9,7 @@ const mockHandleResult = { ticketId: 'ticket-internal-id', displayId: 'TKT-GH01', isNewTicket: true, + isOrphanedReply: false, aiJobEnqueued: true, messageId: 'message-internal-id', }; diff --git a/apps/github-app/src/webhooks/discussion-created.ts b/apps/github-app/src/webhooks/discussion-created.ts index ca802aaa..5dd898b8 100644 --- a/apps/github-app/src/webhooks/discussion-created.ts +++ b/apps/github-app/src/webhooks/discussion-created.ts @@ -57,7 +57,7 @@ export async function handleDiscussionCreated( }, }); - // Intentionally no "Ticket TKT-\u2026 created" acknowledgment comment \u2014 see + // Intentionally no "Ticket TKT-… created" acknowledgment comment — see // the matching note in issues-opened.ts. console.log( diff --git a/apps/github-app/src/webhooks/issues-opened.ts b/apps/github-app/src/webhooks/issues-opened.ts index b00b4da8..d0c1d85a 100644 --- a/apps/github-app/src/webhooks/issues-opened.ts +++ b/apps/github-app/src/webhooks/issues-opened.ts @@ -57,9 +57,9 @@ export async function handleIssueOpened( }, }); - // Intentionally no "Ticket TKT-\u2026 created" acknowledgment comment. The + // Intentionally no "Ticket TKT-… created" acknowledgment comment. The // ticket id is internal, and the AI response lands in the same thread - // moments later \u2014 the ack was pure noise on a public issue. + // moments later — the ack was pure noise on a public issue. console.log( `[GitHub App] Created ticket ${result.displayId} for issue ${repository.full_name}#${issue.number}`, diff --git a/apps/slack-bot/src/events/message.ts b/apps/slack-bot/src/events/message.ts index 25c0d01a..3083fb26 100644 --- a/apps/slack-bot/src/events/message.ts +++ b/apps/slack-bot/src/events/message.ts @@ -58,7 +58,7 @@ export function registerMessageHandler(app: App): void { await handler.handle(message); - // No acknowledgment post \u2014 it leaked the internal ticket displayId to + // No acknowledgment post — it leaked the internal ticket displayId to // the channel and added a second bot message for no reporter benefit. // See the matching change in apps/discord-bot/src/events/thread-create.ts. } catch (error) { diff --git a/apps/teams-bot/src/__tests__/message.test.ts b/apps/teams-bot/src/__tests__/message.test.ts index 8d637dbd..8eb559bd 100644 --- a/apps/teams-bot/src/__tests__/message.test.ts +++ b/apps/teams-bot/src/__tests__/message.test.ts @@ -213,6 +213,33 @@ describe('handleMessage', () => { expect(createJob).not.toHaveBeenCalled(); }); + it('stays silent when an orphaned reply creates a ticket', async () => { + // Teams derives isThreadStart from `!activity.replyToId`, so a bare + // mid-conversation message ("thanks, that worked!") arrives as a reply. + // No matching ticket means the shared handler preserves the reply by + // creating a ticket around it — but this is a conversation Outpost was + // never part of, so the bot must post NOTHING: no acknowledgment card, + // and no conversationReference claiming the thread for proactive + // messaging. isNewTicket is true on that fallback, so a handler that + // branches on isNewTicket alone reintroduces the chatter this guards. + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + const context = makeContext({ + replyToId: 'missing-parent-id', + text: 'thanks, that worked!', + }); + await handleMessage(context); + + // The customer's words are still preserved. + expect(prisma.ticket.create).toHaveBeenCalled(); + expect(prisma.message.create).toHaveBeenCalled(); + + // But nothing is answered and nothing is posted. + expect(createJob).not.toHaveBeenCalled(); + expect(context.sendActivity).not.toHaveBeenCalled(); + expect(prisma.ticket.update).not.toHaveBeenCalled(); + }); + it('does not enqueue AI response for team member follow-ups', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue( TICKET as ReturnType extends Promise ? T : never, diff --git a/apps/teams-bot/src/__tests__/tickets.test.ts b/apps/teams-bot/src/__tests__/tickets.test.ts index 53c53979..0f4f3a78 100644 --- a/apps/teams-bot/src/__tests__/tickets.test.ts +++ b/apps/teams-bot/src/__tests__/tickets.test.ts @@ -5,6 +5,7 @@ vi.mock('@copilotkit/outpost/db', () => mockPrisma()); import { findTicketByConversationId, isTeamMember } from '../lib/tickets.js'; import { prisma } from '@copilotkit/outpost/db'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; describe('findTicketByConversationId', () => { beforeEach(() => { @@ -26,6 +27,31 @@ describe('findTicketByConversationId', () => { expect(result).toEqual(ticket); }); + it('queries by the sourceId buildTicketSourceId derives', async () => { + // The reader must go through the shared helper InboundHandler writes + // with. Inlining the conversation ID is harmless only until the + // derivation changes, at which point reader and writer silently disagree. + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + await findTicketByConversationId('conv-abc'); + + expect(prisma.ticket.findFirst).toHaveBeenCalledWith({ + where: { + source: 'TEAMS', + sourceId: buildTicketSourceId(TicketSource.TEAMS, 'conv-abc'), + }, + }); + }); + + it('skips the query entirely for an unaddressable conversation', async () => { + // buildTicketSourceId yields no key for an empty conversation ID. + // A `sourceId: null` filter would match any keyless row, so don't query. + vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); + + expect(await findTicketByConversationId('')).toBeNull(); + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + }); + it('returns null when no ticket found', async () => { vi.mocked(prisma.ticket.findFirst).mockResolvedValue(null); diff --git a/apps/teams-bot/src/handlers/message.ts b/apps/teams-bot/src/handlers/message.ts index 2103d739..c2bebc5d 100644 --- a/apps/teams-bot/src/handlers/message.ts +++ b/apps/teams-bot/src/handlers/message.ts @@ -63,7 +63,15 @@ export async function handleMessage(context: TurnContext): Promise { // Delegate to the shared inbound handler const result = await inboundHandler.handle(message); - if (result.isNewTicket) { + // An orphaned reply also reports isNewTicket: true — a ticket really was + // created — but it is NOT a conversation Outpost opened. Teams sets + // isThreadStart from `!activity.replyToId`, so a bare mid-conversation + // message ("thanks, that worked!") whose thread we have no ticket for + // lands here. Acking it would post "🎫 We've got your question" into a + // thread we were never part of, and storing the conversationReference + // would claim that thread for proactive messaging. Both are skipped; the + // ticket still exists for a human to pick up from the dashboard. + if (result.isNewTicket && !result.isOrphanedReply) { // Store Teams-specific ConversationReference for proactive messaging const conversationReference = { serviceUrl: activity.serviceUrl ?? 'https://smba.trafficmanager.net/teams/', @@ -77,7 +85,20 @@ export async function handleMessage(context: TurnContext): Promise { }, }); - // New ticket: post acknowledgment card + // New ticket: post acknowledgment card. + // + // Teams is deliberately the only platform that still acknowledges. + // Discord, Slack and the GitHub App dropped their ack posts because + // those were plain text that printed the internal ticket displayId + // into a public channel and gave the reporter nothing to act on. + // Neither objection applies here: buildTicketCreatedCard carries no + // displayId (see apps/teams-bot/src/cards/ticket-created-card.ts, + // asserted by cards.test.ts) and an Adaptive Card is a richer surface + // than a plain text post — it tells the reporter which of the two + // things is about to happen, an AI answer or a human follow-up, off + // result.aiJobEnqueued. Known divergence, not an oversight; if the + // card ever starts rendering an identifier, drop this the way the + // other platforms did. const card = buildTicketCreatedCard({ title: truncate(message.content, 200), }); @@ -91,6 +112,10 @@ export async function handleMessage(context: TurnContext): Promise { console.log( `[Teams Bot] Created ticket ${result.displayId} for conversation ${message.threadId}`, ); + } else if (result.isOrphanedReply) { + console.log( + `[Teams Bot] Untracked mid-conversation message from ${message.platformUsername} filed as ticket ${result.displayId} (no ack card, no conversation reference)`, + ); } else { console.log( `[Teams Bot] Message from ${message.platformUsername} appended to ticket ${result.displayId}`, diff --git a/apps/teams-bot/src/lib/tickets.ts b/apps/teams-bot/src/lib/tickets.ts index 968177ba..9dc7f6c6 100644 --- a/apps/teams-bot/src/lib/tickets.ts +++ b/apps/teams-bot/src/lib/tickets.ts @@ -1,13 +1,27 @@ import { prisma } from '@copilotkit/outpost/db'; +import { TicketSource, buildTicketSourceId } from '@copilotkit/outpost/shared'; /** * Find a ticket by its Teams conversation ID (stored as sourceId with source=TEAMS). + * + * The key is built by buildTicketSourceId — the same helper InboundHandler + * stores tickets with — so this lookup can never search for a spelling nothing + * was written under. For Teams that is the conversation ID verbatim today, but + * routing through the helper is what keeps it that way: the next change to the + * derivation moves writer and reader together. + * + * A null key (no conversation ID) means no ticket can carry it, so there is + * nothing to query — returning early also stops a `sourceId: null` filter from + * matching an unrelated keyless row. */ export async function findTicketByConversationId(conversationId: string) { + const sourceId = buildTicketSourceId(TicketSource.TEAMS, conversationId); + if (sourceId === null) return null; + return prisma.ticket.findFirst({ where: { source: 'TEAMS', - sourceId: conversationId, + sourceId, }, }); } diff --git a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts index c260ae21..c6f678d1 100644 --- a/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts +++ b/packages/outpost/shared/src/__tests__/platforms-inbound.test.ts @@ -81,6 +81,7 @@ describe('InboundHandler', () => { const result = await handler.handle(msg); expect(result.isNewTicket).toBe(true); + expect(result.isOrphanedReply).toBe(false); expect(result.ticketId).toBe('ticket-1'); expect(result.displayId).toMatch(/^TKT-/); @@ -290,6 +291,7 @@ describe('InboundHandler', () => { const result = await handler.handle(msg); expect(result.isNewTicket).toBe(false); + expect(result.isOrphanedReply).toBe(false); expect(result.ticketId).toBe('ticket-existing'); expect(result.displayId).toBe('TKT-EXISTIN'); @@ -453,8 +455,10 @@ describe('InboundHandler', () => { const msg = makeInboundMessage({ isThreadStart: false }); const result = await handler.handle(msg); - // Falls back to creating a new ticket + // Falls back to creating a new ticket, flagged as an orphan so + // callers do not treat it as a conversation Outpost opened. expect(result.isNewTicket).toBe(true); + expect(result.isOrphanedReply).toBe(true); expect(prisma.ticket.create).toHaveBeenCalledTimes(1); }); }); @@ -484,6 +488,10 @@ describe('InboundHandler', () => { expect(createJob).not.toHaveBeenCalled(); expect(result.aiJobEnqueued).toBe(false); expect(result.isNewTicket).toBe(true); + // isNewTicket cannot distinguish this from a real thread start, so + // the orphan flag is what platform handlers gate their ack posts on + // (see apps/teams-bot/src/handlers/message.ts). + expect(result.isOrphanedReply).toBe(true); expect(result.messageId).toBe('msg-1'); }); diff --git a/packages/outpost/shared/src/platforms/inbound.ts b/packages/outpost/shared/src/platforms/inbound.ts index 20eb5951..58c33b87 100644 --- a/packages/outpost/shared/src/platforms/inbound.ts +++ b/packages/outpost/shared/src/platforms/inbound.ts @@ -13,6 +13,8 @@ * 3. Orphaned replies (isThreadStart=false with no matching ticket): create the * Ticket + Message so the customer's words are never dropped, but do NOT * enqueue AI_RESPONSE — we never saw the message that opened the conversation. + * Of the callers, only Teams actually reaches this branch; Discord, the GitHub + * App and Slack drop untracked replies before calling in. See handleReply. * 4. Team member detection via ExternalIdentity -> TeamMember lookup * 5. Sequential display ID generation (TKT-XXXXXXXX) */ @@ -135,7 +137,7 @@ export class InboundHandler { */ async handle(message: InboundMessage): Promise { if (message.isThreadStart) { - return this.handleNewTicket(message, { answer: true }); + return this.handleNewTicket(message, { answer: true, orphanedReply: false }); } return this.handleReply(message); } @@ -148,10 +150,17 @@ export class InboundHandler { * the one message Outpost is allowed to answer), `false` for the orphaned- * reply fallback in `handleReply`, where we are creating a ticket around a * mid-conversation message we must not answer. + * + * `orphanedReply` is surfaced on the result as `isOrphanedReply` so callers + * can distinguish "a real thread started" from "we filed a ticket around a + * message in a conversation we were never part of". It is a separate + * decision from `answer` on purpose — a caller must not have to infer one + * from the other — even though today only the orphan path passes + * `answer: false`. */ private async handleNewTicket( message: InboundMessage, - { answer }: { answer: boolean }, + { answer, orphanedReply }: { answer: boolean; orphanedReply: boolean }, ): Promise { const displayId = generateTicketId(); const authorLabel = `${message.platformUsername} (${message.platformUserId})`; @@ -224,6 +233,7 @@ export class InboundHandler { ticketId: ticket.id, displayId, isNewTicket: true, + isOrphanedReply: orphanedReply, aiJobEnqueued, messageId, }; @@ -255,6 +265,26 @@ export class InboundHandler { // customer's words is worse than filing an oddly-titled ticket, and a // human can pick it up from the dashboard. // + // In practice only Teams reaches this branch. Every other caller + // pre-filters an untracked reply and drops it before we are called: + // - Discord: apps/discord-bot/src/events/message-create.ts returns + // early when findTicketByThreadId finds nothing. + // - GitHub: apps/github-app/src/webhooks/issue-comment.ts returns + // early on no ticket, and never routes comments through this + // handler at all (its InboundHandler callers, issues-opened and + // discussion-created, are thread starts only). + // - Slack: apps/slack-bot/src/events/message.ts queries the ticket + // itself and returns when the reply's thread is untracked. + // The web Postmark webhook does honour the principle, but implements + // it locally (see apps/web/src/app/api/webhooks/postmark/route.ts, + // isOrphanedReply) rather than through this path. + // + // So the preservation rationale above is the intent of this handler, + // not the platform-wide behaviour of Outpost today. A reviewer flagged + // the inconsistency; the resolution was to document it rather than + // change three platforms' filtering. Anyone unifying this should + // remove those pre-filters, not weaken this branch. + // // We do NOT answer it. ASSUMPTION, stated so it is reviewable: the // message that opened the real conversation was never seen by us, so // this reply is not "the message that opened the ticket" in the @@ -267,7 +297,16 @@ export class InboundHandler { // already-answered gate in the AI_RESPONSE handler would wave it // straight through and answer a mid-thread "any update?" in a // conversation Outpost was never part of. - return this.handleNewTicket({ ...message, isThreadStart: true }, { answer: false }); + // isOrphanedReply rides back out on the result: isNewTicket is + // true here (a ticket really was created), so a caller that only + // looks at isNewTicket would treat this like a fresh thread start + // and, on Teams, post an acknowledgment card plus claim the + // conversation for proactive messaging — bot chatter in a thread + // we were never part of. + return this.handleNewTicket( + { ...message, isThreadStart: true }, + { answer: false, orphanedReply: true }, + ); } const authorLabel = `${message.platformUsername} (${message.platformUserId})`; @@ -319,6 +358,7 @@ export class InboundHandler { ticketId: ticket.id, displayId: ticket.displayId, isNewTicket: false, + isOrphanedReply: false, aiJobEnqueued: false, messageId: msg.id, }; diff --git a/packages/outpost/shared/src/platforms/types.ts b/packages/outpost/shared/src/platforms/types.ts index 3c52fe75..44c63676 100644 --- a/packages/outpost/shared/src/platforms/types.ts +++ b/packages/outpost/shared/src/platforms/types.ts @@ -161,6 +161,19 @@ export interface InboundResult { displayId: string; /** Whether a new ticket was created (vs reply appended to existing) */ isNewTicket: boolean; + /** + * Whether this ticket was created by the orphaned-reply fallback — a + * mid-thread message arrived, no ticket matched its thread, and a ticket + * was filed around it so the customer's words are not dropped. + * + * `isNewTicket` is still `true` in that case (a ticket genuinely was + * created), so consumers cannot use it to tell a real thread start from an + * orphan. Anything that would be wrong to do in a conversation Outpost was + * never part of — posting an acknowledgment, claiming the thread by storing + * a conversation reference for proactive messaging — must check this flag + * and skip when it is `true`. + */ + isOrphanedReply: boolean; /** Whether an AI_RESPONSE job was enqueued */ aiJobEnqueued: boolean; /** The message record ID that was created, or null if no message was created */ From 24addde80ee9ca7646ddba31562959a4ca1b1961 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:30:15 -0400 Subject: [PATCH 71/83] fix(worker): repair the missing SystemConfig table and make boot failures legible Production could not deploy from 2026-08-07 to 2026-08-12. Every attempt built and pushed an image, then failed the healthcheck eleven times over five minutes with "1/1 replicas never became healthy". The replica serving production was the 2026-08-03 image the whole time. Root cause: the production database was missing the SystemConfig table. _prisma_migrations recorded 0001_init as applied, but the table that migration declares did not exist. Prisma tracks migrations by name, so `migrate deploy` reported "No pending migrations to apply" on every deploy and never revisited it. Every other model and enum in schema.prisma was present -- SystemConfig was the only gap, confirmed by comparing all 22 models against pg_tables. The worker reads SystemConfig at boot through buildSyncEngine (the persisted status / priority / label mapping configs). That read threw, and because it ran at module scope ABOVE the health server, the process exited before anything bound the port. Railway had nothing to report but a timeout, which looks exactly like a broken image. Nine days of a five-minute silence. Three changes, in the order they matter: 1. A repair migration creating SystemConfig. IF NOT EXISTS, because environments whose 0001_init did create the table must no-op rather than fail. Column definitions copied verbatim from the SystemConfig block in 0001_init. The table has already been created directly in production to end the outage, so this migration no-ops there. It matters for staging -- whose worker last deployed 2026-07-24, before this code existed, and would otherwise hit the same wall -- and for any environment created from here. 2. A schema-drift guard in apps/worker/start.sh. `migrate deploy` only compares the migrations directory against _prisma_migrations; it never inspects the real schema, so a migration recorded as applied but never executed is invisible to it. `migrate diff --from-schema-datasource --to-schema-datamodel --exit-code` compares the LIVE DATABASE against schema.prisma and exits non-zero on any difference, which is exactly this class. Verified against the production database: it reports "No difference detected" now that the table exists, and would have named the missing table before. Deliberately fatal rather than a warning. The worker's sync mappings come from the database, and one running against a schema it does not match would write wrong statuses to Linear. Failing the deploy keeps the previous replica up. 3. /health now binds before anything touches the database. Fail-fast is retained -- a worker that could not read its mappings still must not be reported healthy -- but failing is no longer silent. The port binds first, boot state is tracked, and /health answers 503 with the phase and the error message while boot is unfinished or failed. Railway still fails the deploy and keeps the previous replica, so the outcome is unchanged; the difference is that the reason is now visible from a single probe. The payload logic is extracted to apps/worker/src/health.ts because index.ts is a top-level-await module that binds a port and starts polling on import, so its boot path cannot be exercised from a test. buildHealthResponse is pure and pinned by 5 tests: ready reports 200 with the worker snapshot; failed reports 503 carrying the reason; starting reports 503; phase 'ready' with no worker still reports 503; and a failed boot never returns 200 even with a worker snapshot present. Red-green verified -- reverting to the old always-200 behaviour fails 4 of them. Call-site enumeration: - buildHealthResponse / BootState (new, apps/worker/src/health.ts) -- 1 consumer, the /health handler in index.ts. Not exported from a package entry point; the worker app is not a library. - worker and scheduler in index.ts changed from `const` to nullable `let`, since both are now constructed after the health server binds. Two reads: the health handler (guards on `worker` being non-null) and shutdown() (uses optional calls). No other module imports them -- index.ts is the app entry point and exports nothing. - start.sh -- one caller, the Dockerfile CMD. Syntax checked with `sh -n`. Verification: turbo typecheck 10/10, turbo test 19/19 (1770 tests). The drift guard was run against the live production database rather than only reasoned about. Not addressed here: why 0001_init was recorded as applied without creating that table. The likeliest explanation is that the schema was created with `db push` at some point and the migration marked applied, which means the same drift may exist in staging. The guard in (2) turns that from an invisible failure into a loud one, but confirming staging needs a run from inside Railway -- its Postgres has no public URL. --- apps/worker/src/__tests__/health.test.ts | 65 ++++++++++ apps/worker/src/health.ts | 44 +++++++ apps/worker/src/index.ts | 112 +++++++++++------- apps/worker/start.sh | 42 ++++++- .../migration.sql | 23 ++++ 5 files changed, 244 insertions(+), 42 deletions(-) create mode 100644 apps/worker/src/__tests__/health.test.ts create mode 100644 apps/worker/src/health.ts create mode 100644 packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql diff --git a/apps/worker/src/__tests__/health.test.ts b/apps/worker/src/__tests__/health.test.ts new file mode 100644 index 00000000..bd63fff0 --- /dev/null +++ b/apps/worker/src/__tests__/health.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from 'vitest'; +import { buildHealthResponse, type BootState } from '../health.js'; + +const WORKER_HEALTH = { running: true, activeJobs: 2, pollIntervalMs: 1000 }; + +describe('buildHealthResponse', () => { + it('reports 200 with the worker snapshot once boot is ready', () => { + const boot: BootState = { phase: 'ready', error: null }; + + const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH); + + expect(statusCode).toBe(200); + expect(body).toMatchObject({ status: 'ok', running: true, activeJobs: 2 }); + }); + + // The regression this pins: the worker used to await the database at module + // scope, above the health server, so a boot failure exited the process before + // anything bound the port. Railway could only say "1/1 replicas never became + // healthy" — indistinguishable from a broken image, and it hid a missing + // SystemConfig table for nine days. A failed boot must now answer, and the + // answer must carry the reason. + it('reports 503 AND the reason when boot failed', () => { + const boot: BootState = { + phase: 'failed', + error: 'The table `public.SystemConfig` does not exist in the current database.', + }; + + const { statusCode, body } = buildHealthResponse(boot, null); + + expect(statusCode).toBe(503); + expect(body.status).toBe('failed'); + expect(body.error).toContain('SystemConfig'); + }); + + it('reports 503 while boot is still in progress', () => { + const boot: BootState = { phase: 'starting', error: null }; + + const { statusCode, body } = buildHealthResponse(boot, null); + + expect(statusCode).toBe(503); + expect(body).toEqual({ status: 'starting', error: null }); + }); + + // A half-booted worker must not be reported healthy just because the phase + // flag says ready — the snapshot is what proves the worker exists. + it('does not report 200 when the phase is ready but no worker exists', () => { + const boot: BootState = { phase: 'ready', error: null }; + + const { statusCode, body } = buildHealthResponse(boot, null); + + expect(statusCode).toBe(503); + expect(body.status).toBe('ready'); + }); + + // Fail-fast is retained on purpose: a worker whose sync mappings could not be + // read must never be routed to, because it would write wrong statuses to + // Linear. This pins that a failed boot is not quietly downgraded to healthy. + it('never returns 200 for a failed boot, even with a worker snapshot present', () => { + const boot: BootState = { phase: 'failed', error: 'connection refused' }; + + const { statusCode } = buildHealthResponse(boot, WORKER_HEALTH); + + expect(statusCode).toBe(503); + }); +}); diff --git a/apps/worker/src/health.ts b/apps/worker/src/health.ts new file mode 100644 index 00000000..291e34ca --- /dev/null +++ b/apps/worker/src/health.ts @@ -0,0 +1,44 @@ +/** + * Health payload construction, split out from index.ts so it is testable. + * + * index.ts is a top-level-await module with side effects on import (it binds a + * port and starts polling), so its boot behaviour cannot be exercised directly + * from a test. This function holds the part worth pinning: an unbooted or failed + * worker must report 503 WITH a reason, and only a fully booted one reports 200. + */ + +export type BootPhase = 'starting' | 'ready' | 'failed'; + +export interface BootState { + phase: BootPhase; + error: string | null; +} + +export interface HealthResponse { + statusCode: number; + body: Record; +} + +/** + * Build the /health response. + * + * `workerHealth` is the worker's own health snapshot, or null when the worker has + * not been constructed yet. It is passed rather than read so this stays pure. + * + * 503 on a non-ready phase is deliberate. An unbooted worker must not be reported + * healthy — its sync mappings come from the database, and one running against a + * schema it does not match would write wrong statuses to Linear. The value added + * over simply exiting is the body: it names the phase and the error, so a probe + * alone explains the failure. Exiting before binding the port is what made a + * missing SystemConfig table look identical to a broken image for nine days. + */ +export function buildHealthResponse( + boot: BootState, + workerHealth: Record | null, +): HealthResponse { + if (boot.phase === 'ready' && workerHealth) { + return { statusCode: 200, body: { status: 'ok', ...workerHealth } }; + } + + return { statusCode: 503, body: { status: boot.phase, error: boot.error } }; +} diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 13d2b9a3..950b4dd7 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -15,6 +15,9 @@ * - TRACKER_SYNC: Push changes to external trackers * - JOB_CLEANUP: Periodic cleanup of old jobs and sync events * - GITHUB_REACTION_POLL: Poll GitHub reactions on AI comments (no webhook exists) + * + * BOOT ORDER: /health starts listening before anything touches the database, so a + * boot failure is reported rather than merely fatal. See the boot-state block below. */ import http from 'node:http'; @@ -34,29 +37,78 @@ import { handleGithubReactionPoll, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; +import { buildHealthResponse, type BootState } from './health.js'; -// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── +// ─── Boot state ─────────────────────────────────────────────────────────── -// BOOT SEMANTICS — deliberate change. This is a top-level await that performs -// three database reads (the persisted status / priority / label mapping configs) -// before this module finishes evaluating. If the database is unreachable at boot -// the import throws, so the process exits BEFORE the health server below starts -// listening: the container crash-loops with no /health at all rather than coming -// up and reporting itself degraded. +// Fail-fast on a bad boot is still the intent: a worker running with silently +// defaulted sync mappings would write wrong statuses to Linear, so it must not +// report itself healthy. What changed is that failing is no longer SILENT. +// +// This used to be a top-level `await buildSyncEngine()` above the health server, +// so any boot-time database problem killed the process before anything bound the +// port. Railway could only report "1/1 replicas never became healthy", which is +// indistinguishable from a broken image. That cost nine days of undiagnosed +// deploy failures when SystemConfig turned out to be missing from the production +// database: every deploy from 2026-08-07 failed with no usable signal. // -// Fail-fast is the intent — a worker running with silently-defaulted mappings is -// worse than one that is visibly down, since TRACKER_SYNC would then write wrong -// statuses to Linear. Railway's restart policy is the retry mechanism. Note this -// interacts with the /health honesty follow-up (#138): once /health reflects -// worker state, a degraded-but-listening mode becomes a real option and this -// decision is worth revisiting. -const syncEngine = await buildSyncEngine(); +// Now the port binds first and /health answers 503 with the reason while the boot +// is unfinished or failed. Railway still fails the deploy and keeps the previous +// replica — same outcome, diagnosable in seconds instead of days. +const boot: BootState = { phase: 'starting', error: null }; + +let worker: Worker | null = null; +let scheduler: Scheduler | null = null; + +// ─── Health Server ──────────────────────────────────────────────────────── + +const port = parseInt(process.env.PORT ?? process.env.HEALTH_PORT ?? '3003', 10); + +const healthServer = http.createServer((req, res) => { + if (req.url !== '/health') { + res.writeHead(404); + res.end('Not Found'); + return; + } + + const { statusCode, body } = buildHealthResponse( + boot, + worker ? (worker.healthCheck() as unknown as Record) : null, + ); + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); +}); + +healthServer.listen(port, () => { + console.log(`[Worker] Health server listening on port ${port} (boot: ${boot.phase})`); +}); + +// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── + +// Three database reads (the persisted status / priority / label mapping configs). +// A failure here leaves boot.phase === 'failed' and the process alive but +// unhealthy, so the reason reaches /health and the logs instead of vanishing with +// the process. +let syncEngine: Awaited>; +try { + syncEngine = await buildSyncEngine(); +} catch (error) { + boot.error = error instanceof Error ? error.message : String(error); + boot.phase = 'failed'; + console.error( + `[Worker] BOOT FAILED building the sync engine: ${boot.error}\n` + + `[Worker] /health is listening on ${port} and will report 503 with this reason. ` + + `A missing table or column here means the database does not match schema.prisma — ` + + `check the schema-drift guard in apps/worker/start.sh.`, + ); + throw error; +} const handleTrackerSync = createTrackerSyncHandler(syncEngine); // ─── Create Worker ──────────────────────────────────────────────────────── -const worker = new Worker({ +worker = new Worker({ maxConcurrency: 10, pollIntervalMs: 1000, concurrencyByType: { @@ -89,42 +141,22 @@ worker.on(JobType.TRACKER_SYNC, handleTrackerSync); worker.on(JobType.JOB_CLEANUP, handleJobCleanup); worker.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); -// ─── Start Scheduler ────────────────────────────────────────────────────── - -const scheduler = new Scheduler(); - -// ─── Health Server ──────────────────────────────────────────────────────── - -const port = parseInt(process.env.PORT ?? process.env.HEALTH_PORT ?? '3003', 10); - -const healthServer = http.createServer((req, res) => { - if (req.url === '/health') { - const health = worker.healthCheck(); - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok', ...health })); - } else { - res.writeHead(404); - res.end('Not Found'); - } -}); - // ─── Start Everything ───────────────────────────────────────────────────── -healthServer.listen(port, () => { - console.log(`[Worker] Health server listening on port ${port}`); -}); - +scheduler = new Scheduler(); scheduler.start(); worker.start(); +boot.phase = 'ready'; + console.log('[Worker] Worker process started'); // ─── Graceful Shutdown ──────────────────────────────────────────────────── async function shutdown(signal: string): Promise { console.log(`[Worker] Received ${signal}, shutting down...`); - scheduler.stop(); - await worker.stop(); + scheduler?.stop(); + await worker?.stop(); healthServer.close(); await prisma.$disconnect(); console.log('[Worker] Shutdown complete'); diff --git a/apps/worker/start.sh b/apps/worker/start.sh index 8bc128da..28daee61 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -1,6 +1,44 @@ #!/bin/sh set -e + +PRISMA="node /opt/prisma/node_modules/prisma/build/index.js" +SCHEMA="packages/outpost/db/prisma/schema.prisma" + echo "Running database migrations..." -node /opt/prisma/node_modules/prisma/build/index.js migrate deploy --schema packages/outpost/db/prisma/schema.prisma -echo "Migrations complete. Starting worker..." +$PRISMA migrate deploy --schema "$SCHEMA" + +# Schema-drift guard. +# +# `migrate deploy` only compares the migrations directory against the +# _prisma_migrations bookkeeping table. It never inspects the actual schema, so a +# migration recorded as applied but never executed is invisible to it — it +# cheerfully reports "No pending migrations to apply" against a database missing +# the tables that migration declares. +# +# That is exactly what happened: production had 0001_init recorded as applied +# while the SystemConfig table it declares did not exist. The worker read that +# table during boot, threw, and died before binding /health, so every deploy from +# 2026-08-07 failed with nothing but "1/1 replicas never became healthy" — nine +# days of a five-minute healthcheck timeout that looked like a broken image. +# +# `migrate diff` compares the LIVE DATABASE against schema.prisma and exits +# non-zero when they differ, which catches that class. Running it here means the +# deploy fails in seconds with the drifted object named, instead of timing out. +# +# Deliberately fatal rather than a warning: the worker's sync mappings come from +# the database, and one running against a schema it does not match would write +# wrong statuses to Linear. Failing the deploy keeps the previous replica serving. +echo "Checking for schema drift..." +if ! $PRISMA migrate diff \ + --from-schema-datasource "$SCHEMA" \ + --to-schema-datamodel "$SCHEMA" \ + --exit-code; then + echo "" + echo "FATAL: the database does not match schema.prisma (see the diff above)." + echo "A migration may be recorded as applied without having run." + echo "Compare models in schema.prisma against the live tables before redeploying." + exit 1 +fi + +echo "Migrations complete and schema matches. Starting worker..." exec node apps/worker/dist/index.js diff --git a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql new file mode 100644 index 00000000..0c0b9e57 --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql @@ -0,0 +1,23 @@ +-- Repair migration: create SystemConfig where 0001_init did not. +-- +-- Production has 0001_init recorded as applied in _prisma_migrations, but the +-- SystemConfig table it declares does not exist there. Prisma tracks +-- migrations by name, so `migrate deploy` reports "no pending migrations" and +-- will never create it. Every table and enum in schema.prisma except this one +-- is present, so this is the only gap. +-- +-- The worker reads SystemConfig during boot (buildSyncEngine -> the persisted +-- status/priority/label mapping configs) at module scope, before the health +-- server starts listening. The read throws, the process exits, nothing ever +-- binds /health, and Railway's healthcheck reports "1/1 replicas never became +-- healthy". Production has been unable to deploy since 2026-08-07 as a result. +-- +-- IF NOT EXISTS is deliberate: environments whose 0001_init did create the +-- table must no-op rather than fail. Column definitions match the SystemConfig +-- block in 0001_init exactly. +CREATE TABLE IF NOT EXISTS "SystemConfig" ( + "key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "updatedAt" TIMESTAMP(3) NOT NULL, + CONSTRAINT "SystemConfig_pkey" PRIMARY KEY ("key") +); From 5da02eb16fe5cf4f8781e1db8e92e4c0169abb4e Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 08:38:46 -0700 Subject: [PATCH 72/83] fix(worker): make the boot-failure path actually reachable Review of the previous commit turned up four issues, one of which made its headline change inert. 1. The catch around buildSyncEngine rethrew. This module is a top-level-await entry module, so an exception escaping evaluation rejects the module's evaluation promise, Node reports it as uncaught, and the process exits 1 -- a listening HTTP server does not keep it alive. /health never answered a single probe, and the log line promising "will report 503 with this reason" was false. Verified A/B against an unreachable DATABASE_URL: before, the process exits 1 and curl gets connection refused; after, it stays up and serves 503. The whole boot sequence now lives in startWorker() and is awaited in a try/catch that records the reason instead of rethrowing. Worker construction, scheduler.start() and worker.start() sat outside the old try and would have died the same silent way; they are inside it now. Fail-fast is unchanged in outcome: the healthcheck still fails, Railway still fails the deploy and keeps the previous replica. It just says why now. 2. /health echoed the raw exception. Prisma's connectivity errors quote the database host, port and user (P1001, P1000), and the endpoint is unauthenticated. summarizeBootError keeps the message only for the schema-shape codes that name the missing object (P2021/P2022) -- the actual diagnostic payload -- and reduces everything else to error class plus code, leaving the full text to the logs. The live failure turns out to be PrismaClientInitializationError with errorCode undefined, hence the class-name fallback. 3. The drift guard treated `migrate diff` exit 1 (CLI or connectivity failure) the same as exit 2 (drift detected), so a database blip during deploy printed "the database does not match schema.prisma" with no diff above it. It now captures the status and reports the two cases separately. 4. The health test fixture asserted on `activeJobs`, which is not a field of WorkerHealthStatus, and index.ts cast the real snapshot through `unknown`, so nothing checked the shape actually served. buildHealthResponse now takes WorkerHealthStatus | null and the fixture is built from that type. The spread is reordered so the envelope's own `status` cannot be shadowed by a future field of the same name, with a test pinning it. turbo typecheck 10/10, turbo test 10/10, start.sh drift branches exercised at exit 0/1/2 with a stubbed prisma. --- apps/worker/src/__tests__/health.test.ts | 78 ++++++++++++- apps/worker/src/health.ts | 49 +++++++- apps/worker/src/index.ts | 136 ++++++++++++----------- apps/worker/start.sh | 21 +++- 4 files changed, 213 insertions(+), 71 deletions(-) diff --git a/apps/worker/src/__tests__/health.test.ts b/apps/worker/src/__tests__/health.test.ts index bd63fff0..57b365af 100644 --- a/apps/worker/src/__tests__/health.test.ts +++ b/apps/worker/src/__tests__/health.test.ts @@ -1,7 +1,17 @@ import { describe, it, expect } from 'vitest'; -import { buildHealthResponse, type BootState } from '../health.js'; - -const WORKER_HEALTH = { running: true, activeJobs: 2, pollIntervalMs: 1000 }; +import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; +import { buildHealthResponse, summarizeBootError, type BootState } from '../health.js'; + +// Typed as the real contract, so a rename or removal in WorkerHealthStatus fails +// this file instead of leaving it green against a shape /health never serves. +const WORKER_HEALTH: WorkerHealthStatus = { + running: true, + activeJobCount: 2, + activeJobsByType: { AI_RESPONSE: 2 }, + lastPollTime: new Date('2026-08-12T22:00:00.000Z'), + registeredHandlers: ['AI_RESPONSE', 'TRACKER_SYNC'], + upSince: new Date('2026-08-12T21:00:00.000Z'), +}; describe('buildHealthResponse', () => { it('reports 200 with the worker snapshot once boot is ready', () => { @@ -10,7 +20,18 @@ describe('buildHealthResponse', () => { const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH); expect(statusCode).toBe(200); - expect(body).toMatchObject({ status: 'ok', running: true, activeJobs: 2 }); + expect(body).toMatchObject({ status: 'ok', running: true, activeJobCount: 2 }); + }); + + // `status` is the envelope's field. Spreading the snapshot over it would let a + // future WorkerHealthStatus.status redefine "ok" for every probe silently. + it('keeps its own status field even if the snapshot carries one', () => { + const boot: BootState = { phase: 'ready', error: null }; + const shadowed = { ...WORKER_HEALTH, status: 'degraded' } as unknown as WorkerHealthStatus; + + const { body } = buildHealthResponse(boot, shadowed); + + expect(body.status).toBe('ok'); }); // The regression this pins: the worker used to await the database at module @@ -63,3 +84,52 @@ describe('buildHealthResponse', () => { expect(statusCode).toBe(503); }); }); + +// /health is unauthenticated, so whatever lands in boot.error is published. +// Prisma's connectivity errors quote the database host, port and user; its +// schema-shape errors name the missing table, which is the whole diagnostic +// point. These pin that only the second kind survives to the wire. +describe('summarizeBootError', () => { + it('keeps the object name for a missing-table error', () => { + const error = Object.assign( + new Error('The table `public.SystemConfig` does not exist in the current database.'), + { code: 'P2021' }, + ); + + expect(summarizeBootError(error)).toContain('SystemConfig'); + expect(summarizeBootError(error)).toContain('P2021'); + }); + + it('redacts the host and user out of a connectivity error', () => { + const error = Object.assign( + new Error("Can't reach database server at `db.internal.railway.app:5432`"), + { code: 'P1001' }, + ); + + const summary = summarizeBootError(error); + + expect(summary).toContain('P1001'); + expect(summary).not.toContain('db.internal.railway.app'); + expect(summary).not.toContain('5432'); + }); + + // The real shape observed from a live boot against an unreachable database: + // PrismaClientInitializationError, whose errorCode is undefined and whose + // message quotes host and port. The class name is all that may survive. + it('redacts a connection failure that carries no code at all', () => { + const error = new Error("Can't reach database server at `127.0.0.1:59999`"); + error.name = 'PrismaClientInitializationError'; + + const summary = summarizeBootError(error); + + expect(summary).toContain('PrismaClientInitializationError'); + expect(summary).not.toContain('127.0.0.1'); + }); + + it('redacts credentials out of a plain error', () => { + const summary = summarizeBootError(new Error('postgres://user:hunter2@host/db refused')); + + expect(summary).not.toContain('hunter2'); + expect(summary).toContain('logs'); + }); +}); diff --git a/apps/worker/src/health.ts b/apps/worker/src/health.ts index 291e34ca..bdfe4f65 100644 --- a/apps/worker/src/health.ts +++ b/apps/worker/src/health.ts @@ -7,10 +7,13 @@ * worker must report 503 WITH a reason, and only a fully booted one reports 200. */ +import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; + export type BootPhase = 'starting' | 'ready' | 'failed'; export interface BootState { phase: BootPhase; + /** A redacted reason, safe to serve. See {@link summarizeBootError}. */ error: string | null; } @@ -19,6 +22,45 @@ export interface HealthResponse { body: Record; } +/** + * Prisma error codes whose message names the offending schema object and carries + * no connection details, so the raw text is safe to put on /health. P2021 is a + * missing table, P2022 a missing column — exactly the drift class this endpoint + * exists to make visible, and the part worth reading from a probe. + */ +const SAFE_TO_ECHO_CODES = new Set(['P2021', 'P2022']); + +/** + * Reduce a boot exception to a reason that can be served on /health. + * + * /health is unauthenticated, and Prisma's connectivity errors quote the + * database host, port and user back at you — P1001 is "Can't reach database + * server at `host:port`", P1000 names the user. Echoing `error.message` + * verbatim would publish those to anyone who can reach the probe. Only the + * schema-shape codes keep their message; everything else is reduced to its + * code, with the full text left to the logs. + */ +export function summarizeBootError(error: unknown): string { + // Prisma splits this across two properties: PrismaClientKnownRequestError + // carries `code`, PrismaClientInitializationError carries `errorCode` (often + // undefined, which is why the class name is the fallback below). + const raw = (error ?? {}) as { code?: unknown; errorCode?: unknown }; + const code = + typeof raw.code === 'string' + ? raw.code + : typeof raw.errorCode === 'string' + ? raw.errorCode + : null; + + if (code && SAFE_TO_ECHO_CODES.has(code)) { + return `${code}: ${error instanceof Error ? error.message : String(error)}`; + } + + // Class name and code only. Both are stable, neither quotes the connection. + const label = [error instanceof Error ? error.name : 'Error', code].filter(Boolean).join(' '); + return `${label} — see the worker logs for the full error`; +} + /** * Build the /health response. * @@ -34,10 +76,13 @@ export interface HealthResponse { */ export function buildHealthResponse( boot: BootState, - workerHealth: Record | null, + workerHealth: WorkerHealthStatus | null, ): HealthResponse { if (boot.phase === 'ready' && workerHealth) { - return { statusCode: 200, body: { status: 'ok', ...workerHealth } }; + // `status` last on purpose: it is the envelope's own field, and spreading + // the snapshot over it would let a future WorkerHealthStatus.status + // silently redefine what "ok" means to every probe. + return { statusCode: 200, body: { ...workerHealth, status: 'ok' } }; } return { statusCode: 503, body: { status: boot.phase, error: boot.error } }; diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 950b4dd7..d97316a1 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -37,7 +37,7 @@ import { handleGithubReactionPoll, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; -import { buildHealthResponse, type BootState } from './health.js'; +import { buildHealthResponse, summarizeBootError, type BootState } from './health.js'; // ─── Boot state ─────────────────────────────────────────────────────────── @@ -71,10 +71,7 @@ const healthServer = http.createServer((req, res) => { return; } - const { statusCode, body } = buildHealthResponse( - boot, - worker ? (worker.healthCheck() as unknown as Record) : null, - ); + const { statusCode, body } = buildHealthResponse(boot, worker ? worker.healthCheck() : null); res.writeHead(statusCode, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(body)); }); @@ -83,74 +80,87 @@ healthServer.listen(port, () => { console.log(`[Worker] Health server listening on port ${port} (boot: ${boot.phase})`); }); -// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── +// ─── Boot ───────────────────────────────────────────────────────────────── + +// Everything that can throw at boot lives in here: buildSyncEngine's three +// database reads (the persisted status / priority / label mapping configs), the +// Worker construction, and the scheduler/worker start. Anything that escapes +// leaves boot.phase === 'failed' and the process ALIVE but unhealthy, so the +// reason reaches /health instead of vanishing with the process. +async function startWorker(): Promise { + const syncEngine = await buildSyncEngine(); + const handleTrackerSync = createTrackerSyncHandler(syncEngine); + + const started = new Worker({ + maxConcurrency: 10, + pollIntervalMs: 1000, + concurrencyByType: { + [JobType.AI_RESPONSE]: 4, + [JobType.ESCALATION]: 2, + [JobType.SLA_CHECK]: 1, + [JobType.ONBOARDING_DIGEST]: 1, + [JobType.ACCOUNT_SCORING]: 1, + [JobType.HUBSPOT_SYNC]: 1, + [JobType.TRACKER_SYNC]: 1, + [JobType.JOB_CLEANUP]: 1, + [JobType.GITHUB_REACTION_POLL]: 1, + }, + jobTimeouts: { + [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow + [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large + [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts + }, + }); + + // ─── Register Handlers ──────────────────────────────────────────────── + started.on(JobType.AI_RESPONSE, handleAiResponse); + started.on(JobType.ESCALATION, handleEscalation); + started.on(JobType.SLA_CHECK, handleSlaCheck); + started.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); + started.on(JobType.ACCOUNT_SCORING, handleAccountScoring); + started.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); + started.on(JobType.TRACKER_SYNC, handleTrackerSync); + started.on(JobType.JOB_CLEANUP, handleJobCleanup); + started.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); + + // Published before start() so a probe landing mid-start sees the real worker, + // and so shutdown can stop it if a signal arrives during boot. + worker = started; + scheduler = new Scheduler(); + + scheduler.start(); + started.start(); +} -// Three database reads (the persisted status / priority / label mapping configs). -// A failure here leaves boot.phase === 'failed' and the process alive but -// unhealthy, so the reason reaches /health and the logs instead of vanishing with -// the process. -let syncEngine: Awaited>; +// NOTHING IS RETHROWN HERE, deliberately. This is a top-level-await entry +// module: an exception escaping module evaluation rejects its evaluation +// promise, which Node reports as an uncaught exception and exits on — a +// listening HTTP server does not keep the process alive. Rethrowing would kill +// the health server before it could answer a single probe and hand Railway the +// same bare "1/1 replicas never became healthy" that hid a missing SystemConfig +// table for nine days. Staying up and answering 503 IS the fix. +// +// Fail-fast is still the intent: a worker whose sync mappings could not be read +// must never be reported healthy, because TRACKER_SYNC would write wrong +// statuses to Linear. Railway fails the deploy on the failing healthcheck and +// keeps the previous replica serving — same outcome, with a reason attached. try { - syncEngine = await buildSyncEngine(); + await startWorker(); + boot.phase = 'ready'; + console.log('[Worker] Worker process started'); } catch (error) { - boot.error = error instanceof Error ? error.message : String(error); + boot.error = summarizeBootError(error); boot.phase = 'failed'; + // The full error goes to the logs only — /health carries the redacted form, + // since Prisma's connectivity errors quote the database host, port and user. + console.error('[Worker] BOOT FAILED:', error); console.error( - `[Worker] BOOT FAILED building the sync engine: ${boot.error}\n` + - `[Worker] /health is listening on ${port} and will report 503 with this reason. ` + + `[Worker] The process stays up so /health on ${port} reports 503 ("${boot.error}"). ` + `A missing table or column here means the database does not match schema.prisma — ` + `check the schema-drift guard in apps/worker/start.sh.`, ); - throw error; } -const handleTrackerSync = createTrackerSyncHandler(syncEngine); - -// ─── Create Worker ──────────────────────────────────────────────────────── - -worker = new Worker({ - maxConcurrency: 10, - pollIntervalMs: 1000, - concurrencyByType: { - [JobType.AI_RESPONSE]: 4, - [JobType.ESCALATION]: 2, - [JobType.SLA_CHECK]: 1, - [JobType.ONBOARDING_DIGEST]: 1, - [JobType.ACCOUNT_SCORING]: 1, - [JobType.HUBSPOT_SYNC]: 1, - [JobType.TRACKER_SYNC]: 1, - [JobType.JOB_CLEANUP]: 1, - [JobType.GITHUB_REACTION_POLL]: 1, - }, - jobTimeouts: { - [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow - [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large - [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts - }, -}); - -// ─── Register Handlers ──────────────────────────────────────────────────── - -worker.on(JobType.AI_RESPONSE, handleAiResponse); -worker.on(JobType.ESCALATION, handleEscalation); -worker.on(JobType.SLA_CHECK, handleSlaCheck); -worker.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); -worker.on(JobType.ACCOUNT_SCORING, handleAccountScoring); -worker.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); -worker.on(JobType.TRACKER_SYNC, handleTrackerSync); -worker.on(JobType.JOB_CLEANUP, handleJobCleanup); -worker.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); - -// ─── Start Everything ───────────────────────────────────────────────────── - -scheduler = new Scheduler(); -scheduler.start(); -worker.start(); - -boot.phase = 'ready'; - -console.log('[Worker] Worker process started'); - // ─── Graceful Shutdown ──────────────────────────────────────────────────── async function shutdown(signal: string): Promise { diff --git a/apps/worker/start.sh b/apps/worker/start.sh index 28daee61..fd523d61 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -29,15 +29,32 @@ $PRISMA migrate deploy --schema "$SCHEMA" # the database, and one running against a schema it does not match would write # wrong statuses to Linear. Failing the deploy keeps the previous replica serving. echo "Checking for schema drift..." -if ! $PRISMA migrate diff \ +set +e +$PRISMA migrate diff \ --from-schema-datasource "$SCHEMA" \ --to-schema-datamodel "$SCHEMA" \ - --exit-code; then + --exit-code +DRIFT_STATUS=$? +set -e + +# --exit-code has three outcomes: 0 no difference, 2 a difference, and anything +# else the CLI itself failing (database unreachable, bad DATABASE_URL, schema +# engine did not start). Those are different problems and must not be reported +# with the same message — a script whose whole purpose is naming the real cause +# should not send the on-call hunting for drift that was never detected. +if [ "$DRIFT_STATUS" -eq 2 ]; then echo "" echo "FATAL: the database does not match schema.prisma (see the diff above)." echo "A migration may be recorded as applied without having run." echo "Compare models in schema.prisma against the live tables before redeploying." exit 1 +elif [ "$DRIFT_STATUS" -ne 0 ]; then + echo "" + echo "FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." + echo "This is a tool or connectivity failure, NOT confirmed drift: the database" + echo "was never successfully compared. Check DATABASE_URL and that the database" + echo "is reachable from this container, then redeploy." + exit 1 fi echo "Migrations complete and schema matches. Starting worker..." From 14278a892dcf73c0927f276dee00048a3d75316a Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 09:55:19 -0700 Subject: [PATCH 73/83] fix(worker): make shutdown survive a signal during boot Follow-up to the boot restructure. Two problems, the second found by testing the first. Signal handlers were registered after the boot await, so a SIGTERM arriving while module evaluation was still suspended found no handler and killed the process outright. That window used to be short; now that a failed boot parks the process alive on a 503, it is exactly when Railway tears a bad deploy down. Registration moves above the boot block. worker/scheduler are still null in that window, so the optional calls no-op and shutdown reduces to closing the port and dropping the Prisma connection. Moving it up surfaced a latent bug: `process.on('SIGTERM', () => shutdown(...))` never handled shutdown's returned promise. Signalled mid-boot, $disconnect() rejects with P2024 while tearing down a pool that never filled, and the unhandled rejection killed the process with a stack trace mid-shutdown -- replacing a clean stop with a crash. shutdown() now catches, reports the failed stop and exits non-zero rather than claiming success, guards against a second signal re-entering, and carries a 10s unref'd watchdog so a hung stop() or $disconnect() cannot outlive Railway's grace period. Verified against a hanging database (240.0.0.1, connect never completes): SIGTERM mid-boot now logs "Shutdown failed: " and exits 1 in 7s with no unhandled rejection, where before it dumped a PrismaClientInitializationError stack. SIGTERM after a failed boot shuts down cleanly in 1s and exits 0. turbo typecheck 10/10, turbo test 10/10. --- apps/worker/src/index.ts | 62 ++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index d97316a1..71c9fd1e 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -80,6 +80,53 @@ healthServer.listen(port, () => { console.log(`[Worker] Health server listening on port ${port} (boot: ${boot.phase})`); }); +// ─── Graceful Shutdown ──────────────────────────────────────────────────── + +// Registered BEFORE the boot await, not after it. Boot is the slowest thing this +// process does and can now sit in `starting` or `failed` indefinitely, which is +// exactly when Railway tears a bad deploy down — and a SIGTERM arriving while +// module evaluation is still suspended would find no handler and kill the +// process outright. `worker`/`scheduler` are still null in that window, so the +// optional calls below no-op and this reduces to closing the port and dropping +// the Prisma connection. +let shuttingDown = false; + +async function shutdown(signal: string): Promise { + if (shuttingDown) return; + shuttingDown = true; + console.log(`[Worker] Received ${signal} in boot phase '${boot.phase}', shutting down...`); + + // Nothing below may outlive Railway's stop grace period. Signalled mid-boot, + // $disconnect() waits on a pool that never filled, which turns a clean stop + // into a SIGKILL. unref'd so it never keeps an otherwise-idle process up. + const watchdog = setTimeout(() => { + console.error('[Worker] Shutdown did not finish in 10s, exiting anyway'); + process.exit(1); + }, 10_000); + watchdog.unref(); + + try { + scheduler?.stop(); + await worker?.stop(); + healthServer.close(); + await prisma.$disconnect(); + console.log('[Worker] Shutdown complete'); + process.exit(0); + } catch (error) { + // These reject in practice: a SIGTERM during boot leaves $disconnect() + // tearing down a connection that was never established (P2024). Without + // this the rejection is unhandled and the process dies to a stack trace + // mid-shutdown instead of reporting a failed stop. + console.error('[Worker] Shutdown failed:', error); + process.exit(1); + } +} + +// `void` because an unhandled rejection here would be the very failure the catch +// above exists to prevent. +process.on('SIGTERM', () => void shutdown('SIGTERM')); +process.on('SIGINT', () => void shutdown('SIGINT')); + // ─── Boot ───────────────────────────────────────────────────────────────── // Everything that can throw at boot lives in here: buildSyncEngine's three @@ -160,18 +207,3 @@ try { `check the schema-drift guard in apps/worker/start.sh.`, ); } - -// ─── Graceful Shutdown ──────────────────────────────────────────────────── - -async function shutdown(signal: string): Promise { - console.log(`[Worker] Received ${signal}, shutting down...`); - scheduler?.stop(); - await worker?.stop(); - healthServer.close(); - await prisma.$disconnect(); - console.log('[Worker] Shutdown complete'); - process.exit(0); -} - -process.on('SIGTERM', () => shutdown('SIGTERM')); -process.on('SIGINT', () => shutdown('SIGINT')); From b8a68e3864794dc983aeb7cd4b87a0fa78ab924d Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 10:16:08 -0700 Subject: [PATCH 74/83] fix(worker): harden boot, /health honesty, and the drift guard Round 1 of the 7-agent CR on the two prior commits. Findings converged hard: six of six agents that looked at /health found the same 200-when-not-healthy bug, five of five found the port and bind-error crashes. Most of what follows is a defect in the two commits I added today, not in the original hotfix. BOOT / SHUTDOWN LIFECYCLE - Boot could resume AFTER shutdown began. shutdown() and startWorker() are independent promise chains and nothing sequenced them, so a SIGTERM during buildSyncEngine() let the boot come back behind it: Scheduler.start() ticks every definition immediately and Worker.start() begins claiming jobs, and the pending process.exit then stranded freshly-claimed rows in PROCESSING. startWorker() now checks the shutdown flag after the await. - Boot failure was permanent. Railway's healthcheckPath gates a NEW DEPLOYMENT and does not restart a running service; restartPolicyType="ALWAYS" is restart-on-exit and can never fire on a process that never exits. So a 20s Postgres failover during an ordinary container restart wedged the worker at zero jobs until a human noticed -- strictly worse than the crash-loop it replaced. The reason is now published for BOOT_FAILURE_LINGER_MS and then the process exits 1 so the restart policy retries. Diagnosable and self-healing. - watchdog.unref() disabled the watchdog in exactly its motivating case: with the server closed and $disconnect() hung, no referenced handle remains, so Node exited 0 reporting a clean stop for a shutdown that never finished. Verified with a timer that never fires unref'd and fires ref'd. - The 10s watchdog was shorter than jobTimeouts (300s) that Worker.stop() waits on, so any deploy landing mid-job was guaranteed a forced exit(1). Sized above the drain it guards. - healthServer.close() ran after the drain, advertising a healthy worker for up to 300s after it had committed to dying. Closed first now. - A partially-started boot left the scheduler's setInterval timers running. Torn down through locals in startWorker(). PATHS THAT STILL DIED BEFORE /health COULD ANSWER - PORT="" (a cleared platform variable) parsed to NaN and listen(NaN) throws ERR_SOCKET_BAD_PORT synchronously at module scope -- reproducing the exact opaque failure this PR exists to remove. Extracted resolvePort() with validation and a logged fallback; unit-tested. - healthServer had no 'error' listener, so EADDRINUSE/EACCES was an uncaught exception. It is the one failure that genuinely cannot be reported over the port, so it is now loud in the logs and exits deliberately. - The /health handler was unguarded; worker.healthCheck() or JSON.stringify throwing would have let a probe kill the process it exists to observe. - No unhandledRejection/uncaughtException handlers, so one stray rejection reverted the whole stay-alive design. Added as last-resort handlers. - The BOOT ORDER comment claimed /health binds before anything touches the database. The `prisma` import constructs a PrismaClient at module scope, so that was only ever true of query-time failures. Comment corrected. /health HONESTY - 200 was gated on the snapshot EXISTING, never on running. Worker.stop() sets running=false without exiting (including from Worker's own signal handlers), and a poll blocked on a hung database call freezes lastPollTime -- both left a worker processing nothing while answering 200 {"status":"ok"}. Now requires the worker to actually be polling, with distinct stopped/stalled reasons. - BootState made {phase:'failed', error:null} representable -- a reasonless 503, the exact bug being fixed -- unreachable only by assignment ordering. Now a discriminated union. Every non-200 carries a reason; no body says "ready" on a failure response. - P2021/P2022 messages were echoed raw. Prisma prefixes them with an invocation preamble carrying an absolute container path and a source code frame, so the unauthenticated probe published internal layout. Only the backticked object name is lifted out now, length-bounded. DRIFT GUARD - migrate diff is bidirectional and emitted DROP for anything in the database that schema.prisma does not declare -- routinely true while a sibling service is mid-rollout. Combined with restartPolicyType="ALWAYS" that turned any normal schema-advancing deploy into a worker crash-loop. Now fatal only on objects the schema declares and the database lacks (the outage class), with a NOTE for extras. - Added OUTPOST_ALLOW_SCHEMA_DRIFT=1 as an incident escape hatch, [worker] log prefixes (web and worker interleave in Railway logs), and explicit framing for a migrate deploy failure, which was left to bare set -e. - Pinned the Prisma CLI to 6.19.3. It now gates whether the worker may boot, and `prisma@6` floated; npx resolving 7.9.1 during verification rejected the schema outright, which is precisely the failure mode. - Dockerfile HEALTHCHECK hardcoded 3005 while the code prefers PORT; it now follows the same precedence, over 127.0.0.1 rather than localhost. - railway.toml: added healthcheckTimeout so a known-failed boot fails fast instead of waiting out the 300s default. Verified against a throwaway Postgres: all six drift-guard branches (clean, missing object, escape hatch, extra object, unreachable, exit codes); happy path serves 200 with an intact snapshot and shuts down cleanly in 1s; invalid PORT no longer kills the process; a failed boot lingers then exits 1; /health?probe=1 and /health/ answer rather than 404. Tests 12 -> 24. turbo typecheck 10/10, turbo test 10/10. The boot-after-shutdown window could not be manufactured live -- shutdown reached process.exit before boot resumed in both attempts -- but the outcome is pinned: zero jobs enqueued where the unguarded path would have ticked the scheduler. --- apps/worker/Dockerfile | 12 +- apps/worker/railway.toml | 6 + apps/worker/src/__tests__/health.test.ts | 223 +++++++++++++++--- apps/worker/src/health.ts | 198 +++++++++++++--- apps/worker/src/index.ts | 217 +++++++++++++---- apps/worker/start.sh | 85 +++++-- .../migration.sql | 7 + 7 files changed, 605 insertions(+), 143 deletions(-) diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 645b772c..c432f0ce 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -45,14 +45,22 @@ RUN chmod +x apps/worker/start.sh # Prisma CLI for running migrations at startup (installed standalone, mirrors apps/web/Dockerfile — # the pnpm workspace .bin paths don't resolve reliably in the pruned production image). -RUN npm install --prefix /opt/prisma --no-save prisma@6 +# Pinned to the lockfile's version, not the `prisma@6` range: start.sh now branches on +# this CLI's `migrate diff` output to decide whether the worker may boot at all, so a +# floating 6.x could change that contract and block every deploy. Keep in step with +# @prisma/client — a CLI/client mismatch can itself report false drift. +RUN npm install --prefix /opt/prisma --no-save prisma@6.19.3 USER outpost ENV NODE_ENV=production ENV HEALTH_PORT=3005 EXPOSE 3005 +# Follows the same PORT-over-HEALTH_PORT precedence as src/index.ts. Hardcoding 3005 +# meant that whenever the platform injected PORT, the app moved and this probe failed +# forever against a healthy worker. 127.0.0.1 rather than localhost, which can resolve +# to ::1 inside the container and be refused (see apps/web/Dockerfile). HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD wget -qO- http://localhost:3005/health || exit 1 + CMD wget -qO- "http://127.0.0.1:${PORT:-${HEALTH_PORT:-3005}}/health" || exit 1 CMD ["./apps/worker/start.sh"] diff --git a/apps/worker/railway.toml b/apps/worker/railway.toml index d99b38bf..0dd66015 100644 --- a/apps/worker/railway.toml +++ b/apps/worker/railway.toml @@ -9,3 +9,9 @@ dockerfilePath = "apps/worker/Dockerfile" [deploy] restartPolicyType = "ALWAYS" healthcheckPath = "/health" +# A failed boot now answers 503 with its reason immediately rather than dying +# silently, so there is nothing to gain from Railway's 300s default — that +# five-minute wait per attempt is what made the 2026-08-07 incident so slow to +# read. Kept above the worker's BOOT_FAILURE_LINGER_MS (120s) so the probe sees +# the reason before the process exits to be restarted. +healthcheckTimeout = 180 diff --git a/apps/worker/src/__tests__/health.test.ts b/apps/worker/src/__tests__/health.test.ts index 57b365af..770eac60 100644 --- a/apps/worker/src/__tests__/health.test.ts +++ b/apps/worker/src/__tests__/health.test.ts @@ -1,6 +1,14 @@ import { describe, it, expect } from 'vitest'; import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; -import { buildHealthResponse, summarizeBootError, type BootState } from '../health.js'; +import { + buildHealthResponse, + resolvePort, + summarizeBootError, + STALE_POLL_MS, + type BootState, +} from '../health.js'; + +const NOW = new Date('2026-08-12T22:00:00.000Z').getTime(); // Typed as the real contract, so a rename or removal in WorkerHealthStatus fails // this file instead of leaving it green against a shape /health never serves. @@ -8,16 +16,14 @@ const WORKER_HEALTH: WorkerHealthStatus = { running: true, activeJobCount: 2, activeJobsByType: { AI_RESPONSE: 2 }, - lastPollTime: new Date('2026-08-12T22:00:00.000Z'), + lastPollTime: new Date(NOW - 1_000), registeredHandlers: ['AI_RESPONSE', 'TRACKER_SYNC'], - upSince: new Date('2026-08-12T21:00:00.000Z'), + upSince: new Date(NOW - 3_600_000), }; describe('buildHealthResponse', () => { - it('reports 200 with the worker snapshot once boot is ready', () => { - const boot: BootState = { phase: 'ready', error: null }; - - const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH); + it('reports 200 with the worker snapshot once boot is ready and the worker is polling', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); expect(statusCode).toBe(200); expect(body).toMatchObject({ status: 'ok', running: true, activeJobCount: 2 }); @@ -26,10 +32,9 @@ describe('buildHealthResponse', () => { // `status` is the envelope's field. Spreading the snapshot over it would let a // future WorkerHealthStatus.status redefine "ok" for every probe silently. it('keeps its own status field even if the snapshot carries one', () => { - const boot: BootState = { phase: 'ready', error: null }; const shadowed = { ...WORKER_HEALTH, status: 'degraded' } as unknown as WorkerHealthStatus; - const { body } = buildHealthResponse(boot, shadowed); + const { body } = buildHealthResponse({ phase: 'ready' }, shadowed, NOW); expect(body.status).toBe('ok'); }); @@ -43,61 +48,127 @@ describe('buildHealthResponse', () => { it('reports 503 AND the reason when boot failed', () => { const boot: BootState = { phase: 'failed', - error: 'The table `public.SystemConfig` does not exist in the current database.', + error: 'P2021: missing database object `public.SystemConfig`', }; - const { statusCode, body } = buildHealthResponse(boot, null); + const { statusCode, body } = buildHealthResponse(boot, null, NOW); expect(statusCode).toBe(503); expect(body.status).toBe('failed'); expect(body.error).toContain('SystemConfig'); }); - it('reports 503 while boot is still in progress', () => { - const boot: BootState = { phase: 'starting', error: null }; + it('reports 503 with a reason while boot is still in progress', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'starting' }, null, NOW); - const { statusCode, body } = buildHealthResponse(boot, null); + expect(statusCode).toBe(503); + expect(body.status).toBe('starting'); + expect(body.error).toBeTruthy(); + }); + + // Fail-fast is retained on purpose: a worker whose sync mappings could not be + // read must never be routed to, because it would write wrong statuses to + // Linear. This pins that a failed boot is not quietly downgraded to healthy. + it('never returns 200 for a failed boot, even with a worker snapshot present', () => { + const boot: BootState = { phase: 'failed', error: 'connection refused' }; + + const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH, NOW); expect(statusCode).toBe(503); - expect(body).toEqual({ status: 'starting', error: null }); + expect(body.error).toBe('connection refused'); }); - // A half-booted worker must not be reported healthy just because the phase - // flag says ready — the snapshot is what proves the worker exists. - it('does not report 200 when the phase is ready but no worker exists', () => { - const boot: BootState = { phase: 'ready', error: null }; + // The gap that actually reaches production. Worker.stop() sets running=false + // without exiting the process — including from Worker's OWN signal handlers — + // so gating the 200 on the snapshot's existence alone answered + // 200 {"status":"ok","running":false} for a worker processing nothing. + it('does not report 200 for a worker that has stopped', () => { + const stopped: WorkerHealthStatus = { ...WORKER_HEALTH, running: false, upSince: null }; - const { statusCode, body } = buildHealthResponse(boot, null); + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stopped, NOW); expect(statusCode).toBe(503); - expect(body.status).toBe('ready'); + expect(body.status).toBe('stopped'); + expect(body.error).toContain('not running'); }); - // Fail-fast is retained on purpose: a worker whose sync mappings could not be - // read must never be routed to, because it would write wrong statuses to - // Linear. This pins that a failed boot is not quietly downgraded to healthy. - it('never returns 200 for a failed boot, even with a worker snapshot present', () => { - const boot: BootState = { phase: 'failed', error: 'connection refused' }; + // Worker.poll() catches every error and reschedules, so a poll blocked on a + // hung database call leaves running=true forever with lastPollTime frozen. + it('does not report 200 for a worker whose poll loop has stalled', () => { + const stalled: WorkerHealthStatus = { + ...WORKER_HEALTH, + lastPollTime: new Date(NOW - STALE_POLL_MS - 1), + }; + + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stalled, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('stalled'); + }); + + it('does not report 200 for a worker that has never polled', () => { + const neverPolled: WorkerHealthStatus = { ...WORKER_HEALTH, lastPollTime: null }; - const { statusCode } = buildHealthResponse(boot, WORKER_HEALTH); + const { statusCode } = buildHealthResponse({ phase: 'ready' }, neverPolled, NOW); expect(statusCode).toBe(503); }); + + // A half-booted worker must not be reported healthy just because the phase + // flag says ready — and the 503 must still explain itself rather than + // answering {"status":"ready","error":null}, which is the reasonless body + // this endpoint exists to eliminate. + it('reports 503 with a reason when the phase is ready but no worker exists', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, null, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('no-worker'); + expect(body.error).toContain('boot sequence'); + }); + + it('serves a body that survives JSON serialization', () => { + const { body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); + + expect(() => JSON.stringify(body)).not.toThrow(); + expect(JSON.parse(JSON.stringify(body))).toMatchObject({ status: 'ok', running: true }); + }); }); // /health is unauthenticated, so whatever lands in boot.error is published. // Prisma's connectivity errors quote the database host, port and user; its -// schema-shape errors name the missing table, which is the whole diagnostic -// point. These pin that only the second kind survives to the wire. +// schema errors arrive as a multi-line blob whose preamble carries an absolute +// container path and a source code frame. Only the object name may survive. describe('summarizeBootError', () => { - it('keeps the object name for a missing-table error', () => { + // The shape Prisma actually throws — not a hand-built single-line message. + const realisticP2021 = Object.assign( + new Error( + 'Invalid `prisma.systemConfig.findUnique()` invocation in\n' + + '/app/packages/outpost/shared/dist/sync/config.js:34:56\n\n' + + ' 31 const existing = await db.systemConfig.findUnique({\n\n' + + 'The table `public.SystemConfig` does not exist in the current database.', + ), + { code: 'P2021' }, + ); + + it('names the missing object without leaking container paths or the code frame', () => { + const summary = summarizeBootError(realisticP2021); + + expect(summary).toContain('P2021'); + expect(summary).toContain('public.SystemConfig'); + expect(summary).not.toContain('/app/'); + expect(summary).not.toContain('findUnique'); + }); + + it('covers P2022 missing-column drift, not just P2021', () => { const error = Object.assign( - new Error('The table `public.SystemConfig` does not exist in the current database.'), - { code: 'P2021' }, + new Error( + 'The column `public.SystemConfig.updatedAt` does not exist in the current database.', + ), + { code: 'P2022' }, ); - expect(summarizeBootError(error)).toContain('SystemConfig'); - expect(summarizeBootError(error)).toContain('P2021'); + expect(summarizeBootError(error)).toContain('P2022'); + expect(summarizeBootError(error)).toContain('SystemConfig.updatedAt'); }); it('redacts the host and user out of a connectivity error', () => { @@ -113,10 +184,20 @@ describe('summarizeBootError', () => { expect(summary).not.toContain('5432'); }); - // The real shape observed from a live boot against an unreachable database: - // PrismaClientInitializationError, whose errorCode is undefined and whose - // message quotes host and port. The class name is all that may survive. - it('redacts a connection failure that carries no code at all', () => { + // PrismaClientInitializationError carries `errorCode`, not `code` — the real + // shape observed from a live boot against an unreachable database. + it('reads errorCode as well as code', () => { + const error = Object.assign( + new Error('Timed out fetching a new connection from the pool'), + { + errorCode: 'P2024', + }, + ); + + expect(summarizeBootError(error)).toContain('P2024'); + }); + + it('falls back to the error class when no code is present at all', () => { const error = new Error("Can't reach database server at `127.0.0.1:59999`"); error.name = 'PrismaClientInitializationError'; @@ -130,6 +211,70 @@ describe('summarizeBootError', () => { const summary = summarizeBootError(new Error('postgres://user:hunter2@host/db refused')); expect(summary).not.toContain('hunter2'); - expect(summary).toContain('logs'); + }); + + it('handles thrown non-Error values', () => { + expect(summarizeBootError('boom')).toBeTruthy(); + expect(summarizeBootError(null)).toBeTruthy(); + expect(summarizeBootError(undefined)).toBeTruthy(); + }); + + // A plain object carrying a safe code is the one case the echo branch exists + // for; String(obj) would render "[object Object]". + it('reads .message off a non-Error object carrying a safe code', () => { + const summary = summarizeBootError({ + code: 'P2021', + message: 'The table `public.SystemConfig` does not exist in the current database.', + }); + + expect(summary).toContain('public.SystemConfig'); + expect(summary).not.toContain('[object Object]'); + }); + + it('bounds the length of anything it serves', () => { + const error = Object.assign( + new Error(`The table \`${'x'.repeat(5_000)}\` does not exist.`), + { + code: 'P2021', + }, + ); + + expect(summarizeBootError(error).length).toBeLessThanOrEqual(200); + }); +}); + +// An invalid port used to reach server.listen() as NaN, which throws +// ERR_SOCKET_BAD_PORT synchronously at module scope — killing the process before +// anything bound, the exact opaque failure the health server exists to prevent. +describe('resolvePort', () => { + it('prefers PORT, then HEALTH_PORT, then the default', () => { + expect(resolvePort({ PORT: '8080', HEALTH_PORT: '3005' })).toMatchObject({ + port: 8080, + source: 'PORT', + }); + expect(resolvePort({ HEALTH_PORT: '3005' })).toMatchObject({ + port: 3005, + source: 'HEALTH_PORT', + }); + expect(resolvePort({})).toMatchObject({ port: 3003, source: 'default' }); + }); + + // The reported trigger: `??` only falls through on null/undefined, so a + // cleared platform variable arrives as '' and parses to NaN. + it('falls back with a warning on an empty PORT rather than yielding NaN', () => { + const resolved = resolvePort({ PORT: '', HEALTH_PORT: '3005' }); + + expect(resolved.port).toBe(3005); + expect(resolved.source).toBe('HEALTH_PORT'); + }); + + it('falls back with a warning on a non-numeric or out-of-range PORT', () => { + for (const bad of ['tcp://host:5432', 'abc', '70000', '-1']) { + const resolved = resolvePort({ PORT: bad }); + + expect(resolved.port).toBe(3003); + expect(resolved.warning).toContain(bad); + expect(Number.isInteger(resolved.port)).toBe(true); + } }); }); diff --git a/apps/worker/src/health.ts b/apps/worker/src/health.ts index bdfe4f65..7834c4d4 100644 --- a/apps/worker/src/health.ts +++ b/apps/worker/src/health.ts @@ -1,21 +1,28 @@ /** - * Health payload construction, split out from index.ts so it is testable. + * Boot state, health payload construction, and port resolution — split out from + * index.ts so they are testable. * * index.ts is a top-level-await module with side effects on import (it binds a * port and starts polling), so its boot behaviour cannot be exercised directly - * from a test. This function holds the part worth pinning: an unbooted or failed - * worker must report 503 WITH a reason, and only a fully booted one reports 200. + * from a test. Everything here is pure and pinned by tests: an unbooted, failed, + * stopped or stalled worker must report 503 WITH a reason, and only a worker + * that is actually polling reports 200. */ import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; export type BootPhase = 'starting' | 'ready' | 'failed'; -export interface BootState { - phase: BootPhase; - /** A redacted reason, safe to serve. See {@link summarizeBootError}. */ - error: string | null; -} +/** + * Discriminated so the failed-without-a-reason state is unrepresentable. A 503 + * carrying `error: null` is the exact signal-quality bug this module exists to + * remove; making it a type error is cheaper than remembering to assign `error` + * before `phase` on every future edit. + */ +export type BootState = + | { phase: 'starting' } + | { phase: 'ready' } + | { phase: 'failed'; error: string }; export interface HealthResponse { statusCode: number; @@ -23,28 +30,53 @@ export interface HealthResponse { } /** - * Prisma error codes whose message names the offending schema object and carries - * no connection details, so the raw text is safe to put on /health. P2021 is a - * missing table, P2022 a missing column — exactly the drift class this endpoint - * exists to make visible, and the part worth reading from a probe. + * Prisma error codes whose failure names a schema object rather than a + * connection. P2021 is a missing table, P2022 a missing column — the drift class + * this endpoint exists to surface. Only the object name is echoed, never the + * message (see summarizeBootError). */ const SAFE_TO_ECHO_CODES = new Set(['P2021', 'P2022']); +/** Worker.poll() reschedules every 1s, so a minute of silence means it is wedged. */ +export const STALE_POLL_MS = 60_000; + +/** Upper bound on any reason string served to an unauthenticated probe. */ +const MAX_REASON_LENGTH = 200; + +function messageOf(error: unknown): string { + if (error instanceof Error) return error.message; + if ( + typeof error === 'object' && + error !== null && + typeof (error as { message?: unknown }).message === 'string' + ) { + return (error as { message: string }).message; + } + return String(error); +} + /** * Reduce a boot exception to a reason that can be served on /health. * - * /health is unauthenticated, and Prisma's connectivity errors quote the - * database host, port and user back at you — P1001 is "Can't reach database - * server at `host:port`", P1000 names the user. Echoing `error.message` - * verbatim would publish those to anyone who can reach the probe. Only the - * schema-shape codes keep their message; everything else is reduced to its - * code, with the full text left to the logs. + * /health is unauthenticated. Prisma's connectivity errors quote the database + * host, port and user (P1001 names host:port, P1000 names the user), and even + * the "safe" schema errors arrive as a multi-line blob whose preamble carries + * an absolute container path and a source code frame: + * + * Invalid `prisma.systemConfig.findUnique()` invocation in + * /app/packages/outpost/shared/dist/sync/config.js:34:56 + * 31 const existing = await db.systemConfig.findUnique({ + * The table `public.SystemConfig` does not exist in the current database. + * + * So nothing is echoed verbatim. For the schema codes the backticked object name + * is lifted out of the final line — that name is the entire diagnostic payload — + * and everything else degrades to error class plus code, with the full text left + * to the logs. */ export function summarizeBootError(error: unknown): string { - // Prisma splits this across two properties: PrismaClientKnownRequestError - // carries `code`, PrismaClientInitializationError carries `errorCode` (often - // undefined, which is why the class name is the fallback below). const raw = (error ?? {}) as { code?: unknown; errorCode?: unknown }; + // PrismaClientKnownRequestError carries `code`; PrismaClientInitializationError + // carries `errorCode` (frequently undefined, hence the class-name fallback). const code = typeof raw.code === 'string' ? raw.code @@ -53,37 +85,129 @@ export function summarizeBootError(error: unknown): string { : null; if (code && SAFE_TO_ECHO_CODES.has(code)) { - return `${code}: ${error instanceof Error ? error.message : String(error)}`; + const lastLine = messageOf(error) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .at(-1); + const object = lastLine ? /`([^`]+)`/.exec(lastLine)?.[1] : undefined; + if (object) { + return truncate( + `${code}: missing database object \`${object}\` — the database does not match schema.prisma`, + ); + } } // Class name and code only. Both are stable, neither quotes the connection. const label = [error instanceof Error ? error.name : 'Error', code].filter(Boolean).join(' '); - return `${label} — see the worker logs for the full error`; + return truncate(`${label} — see the worker logs for the full error`); +} + +function truncate(reason: string): string { + return reason.length <= MAX_REASON_LENGTH + ? reason + : `${reason.slice(0, MAX_REASON_LENGTH - 1)}…`; +} + +/** + * Resolve the health-server port from the environment. + * + * `??` is not enough: an empty or non-numeric PORT (a cleared platform variable, + * or a reference variable that failed to resolve) parses to NaN, and + * `server.listen(NaN)` throws ERR_SOCKET_BAD_PORT synchronously at module scope + * — killing the process before anything binds, which is precisely the opaque + * "replicas never became healthy" failure this whole module exists to prevent. + * An invalid value falls back to the default and says so. + */ +export function resolvePort( + env: { PORT?: string; HEALTH_PORT?: string }, + fallback = 3003, +): { port: number; source: string; warning: string | null } { + const candidates: Array<[string, string | undefined]> = [ + ['PORT', env.PORT], + ['HEALTH_PORT', env.HEALTH_PORT], + ]; + + for (const [source, raw] of candidates) { + if (raw === undefined || raw.trim() === '') continue; + const parsed = Number.parseInt(raw, 10); + if (Number.isInteger(parsed) && parsed >= 0 && parsed <= 65535) { + return { port: parsed, source, warning: null }; + } + return { + port: fallback, + source: 'default', + warning: `invalid ${source}="${raw}" (expected an integer 0-65535), falling back to ${fallback}`, + }; + } + + return { port: fallback, source: 'default', warning: null }; } /** * Build the /health response. * - * `workerHealth` is the worker's own health snapshot, or null when the worker has - * not been constructed yet. It is passed rather than read so this stays pure. + * `workerHealth` is the worker's own snapshot, or null when the worker has not + * been constructed. It is passed rather than read so this stays pure. + * + * Every non-200 answer carries a reason. The value added over simply exiting is + * that body: a probe alone explains the failure. Exiting before binding the port + * is what made a missing SystemConfig table look identical to a broken image for + * nine days. * - * 503 on a non-ready phase is deliberate. An unbooted worker must not be reported - * healthy — its sync mappings come from the database, and one running against a - * schema it does not match would write wrong statuses to Linear. The value added - * over simply exiting is the body: it names the phase and the error, so a probe - * alone explains the failure. Exiting before binding the port is what made a - * missing SystemConfig table look identical to a broken image for nine days. + * 200 requires the worker to be *polling*, not merely constructed. Worker.stop() + * sets running=false without exiting the process, and a poll blocked on a hung + * database call freezes lastPollTime — both leave a worker that processes + * nothing while looking alive, which is the honesty gap tracked by #138. */ export function buildHealthResponse( boot: BootState, workerHealth: WorkerHealthStatus | null, + now: number = Date.now(), ): HealthResponse { - if (boot.phase === 'ready' && workerHealth) { - // `status` last on purpose: it is the envelope's own field, and spreading - // the snapshot over it would let a future WorkerHealthStatus.status - // silently redefine what "ok" means to every probe. - return { statusCode: 200, body: { ...workerHealth, status: 'ok' } }; + if (boot.phase === 'failed') { + return { statusCode: 503, body: { status: 'failed', error: boot.error } }; + } + + if (boot.phase === 'starting') { + return { statusCode: 503, body: { status: 'starting', error: 'boot has not finished' } }; + } + + if (!workerHealth) { + return { + statusCode: 503, + body: { + status: 'no-worker', + error: 'boot reported ready but no worker was constructed — this is a bug in the boot sequence, not a database problem', + }, + }; + } + + if (!workerHealth.running) { + return { + statusCode: 503, + body: { + ...workerHealth, + status: 'stopped', + error: 'worker is not running — it was stopped without the process exiting', + }, + }; + } + + const sincePoll = workerHealth.lastPollTime ? now - workerHealth.lastPollTime.getTime() : null; + if (sincePoll === null || sincePoll > STALE_POLL_MS) { + return { + statusCode: 503, + body: { + ...workerHealth, + status: 'stalled', + error: `worker has not polled for ${sincePoll ?? 'any'}ms — the poll loop is blocked, most likely on a hung database call`, + }, + }; } - return { statusCode: 503, body: { status: boot.phase, error: boot.error } }; + // `status` last on purpose: it is the envelope's own field, and spreading the + // snapshot over it would let a future WorkerHealthStatus.status silently + // redefine what "ok" means to every probe. + return { statusCode: 200, body: { ...workerHealth, status: 'ok' } }; } diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 71c9fd1e..eb532317 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -16,8 +16,13 @@ * - JOB_CLEANUP: Periodic cleanup of old jobs and sync events * - GITHUB_REACTION_POLL: Poll GitHub reactions on AI comments (no webhook exists) * - * BOOT ORDER: /health starts listening before anything touches the database, so a - * boot failure is reported rather than merely fatal. See the boot-state block below. + * BOOT ORDER: /health starts listening before any database QUERY runs, so a boot + * failure is reported rather than merely fatal. See the boot-state block below. + * Two classes still escape it, both by construction: the `prisma` import below + * constructs a PrismaClient at module scope (it throws for an ungenerated client + * or an unparseable DATABASE_URL), and a failure to bind the port itself cannot + * be reported over the port. Both are handled loudly rather than silently — see + * the health-server error handler and the last-resort handlers at the bottom. */ import http from 'node:http'; @@ -37,7 +42,7 @@ import { handleGithubReactionPoll, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; -import { buildHealthResponse, summarizeBootError, type BootState } from './health.js'; +import { buildHealthResponse, resolvePort, summarizeBootError, type BootState } from './health.js'; // ─── Boot state ─────────────────────────────────────────────────────────── @@ -53,70 +58,135 @@ import { buildHealthResponse, summarizeBootError, type BootState } from './healt // database: every deploy from 2026-08-07 failed with no usable signal. // // Now the port binds first and /health answers 503 with the reason while the boot -// is unfinished or failed. Railway still fails the deploy and keeps the previous -// replica — same outcome, diagnosable in seconds instead of days. -const boot: BootState = { phase: 'starting', error: null }; +// is unfinished or failed, so the reason is one probe away instead of buried in +// container logs nobody had reason to suspect. +// +// A failed boot does NOT park here forever. Railway's healthcheckPath gates a NEW +// DEPLOYMENT; it does not continuously probe and restart an already-running +// service, and restartPolicyType="ALWAYS" is a restart-on-exit policy that can +// never fire on a process that never exits. Staying up indefinitely would mean a +// 20-second Postgres failover during an ordinary container restart wedges the +// worker with zero jobs processed until a human notices — strictly worse than the +// crash-loop it replaced. So the reason is published for BOOT_FAILURE_LINGER_MS +// (long enough for the deploy probe and any log scrape to read it) and then the +// process exits non-zero so the restart policy retries. Diagnosable AND +// self-healing; the two were never actually in tension. +let boot: BootState = { phase: 'starting' }; let worker: Worker | null = null; let scheduler: Scheduler | null = null; // ─── Health Server ──────────────────────────────────────────────────────── -const port = parseInt(process.env.PORT ?? process.env.HEALTH_PORT ?? '3003', 10); +const { port, source: portSource, warning: portWarning } = resolvePort(process.env); +if (portWarning) console.error(`[Worker] ${portWarning}`); const healthServer = http.createServer((req, res) => { - if (req.url !== '/health') { - res.writeHead(404); - res.end('Not Found'); - return; + // A probe must never be able to kill the process it exists to observe: + // worker.healthCheck() and JSON.stringify both run in this callback, and an + // exception in an http listener is an uncaught exception. + try { + const path = + new URL(req.url ?? '/', 'http://localhost').pathname.replace(/\/+$/, '') || '/'; + if (path !== '/health') { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not Found'); + return; + } + + const { statusCode, body } = buildHealthResponse( + boot, + worker ? worker.healthCheck() : null, + ); + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); + } catch (error) { + console.error('[Worker] /health handler threw:', error); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + status: 'error', + error: 'health handler failed — see the worker logs', + }), + ); } +}); - const { statusCode, body } = buildHealthResponse(boot, worker ? worker.healthCheck() : null); - res.writeHead(statusCode, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(body)); +// listen() reports bind failures asynchronously through 'error'. With no listener +// that is an uncaught exception: the port never binds and the process dies to a +// bare stack trace — the same opaque signal as the original incident, arriving +// through the one step everything else now depends on. It is also the single +// failure that genuinely cannot be reported over /health, so it must be loud in +// the logs and must exit rather than linger pretending to serve. +healthServer.on('error', (error: NodeJS.ErrnoException) => { + console.error( + `[Worker] FATAL: could not bind the health server to port ${port} (${error.code ?? 'unknown'}, from ${portSource}). ` + + `Nothing can report this process's state without it. Check PORT/HEALTH_PORT and whether another process holds the port.`, + error, + ); + process.exit(1); }); healthServer.listen(port, () => { - console.log(`[Worker] Health server listening on port ${port} (boot: ${boot.phase})`); + console.log( + `[Worker] Health server listening on port ${port} (from ${portSource}, boot: ${boot.phase})`, + ); }); // ─── Graceful Shutdown ──────────────────────────────────────────────────── // Registered BEFORE the boot await, not after it. Boot is the slowest thing this -// process does and can now sit in `starting` or `failed` indefinitely, which is -// exactly when Railway tears a bad deploy down — and a SIGTERM arriving while -// module evaluation is still suspended would find no handler and kill the -// process outright. `worker`/`scheduler` are still null in that window, so the -// optional calls below no-op and this reduces to closing the port and dropping -// the Prisma connection. +// process does, which is exactly when Railway tears a bad deploy down — and a +// SIGTERM arriving while module evaluation is still suspended would find no +// handler and kill the process outright. +// +// Note there is a SECOND registrar: Worker.start() installs its own SIGTERM / +// SIGINT handlers that call worker.stop() unawaited. Both fire. That is safe +// only because Worker.stop() early-returns on !running and this handler runs +// first, so ordering here is load-bearing — do not move this registration below +// startWorker(). let shuttingDown = false; +// Longer than the largest entry in jobTimeouts below (300s), because +// Worker.stop() waits for in-flight jobs to finish. A watchdog shorter than the +// drain it guards would turn every deploy that lands mid-job into a forced +// exit(1) — guarding the hang while breaking the normal path. +const SHUTDOWN_WATCHDOG_MS = Number(process.env.SHUTDOWN_WATCHDOG_MS ?? 330_000); + async function shutdown(signal: string): Promise { if (shuttingDown) return; shuttingDown = true; console.log(`[Worker] Received ${signal} in boot phase '${boot.phase}', shutting down...`); - // Nothing below may outlive Railway's stop grace period. Signalled mid-boot, - // $disconnect() waits on a pool that never filled, which turns a clean stop - // into a SIGKILL. unref'd so it never keeps an otherwise-idle process up. + // NOT unref'd. The motivating case is a $disconnect() that never settles + // after the server is closed — precisely when no other referenced handle + // remains, so an unref'd timer would let Node exit 0 (reporting a clean stop + // for a shutdown that never completed) and this line would never print. + // Every path below ends in process.exit, so a referenced timer costs nothing. const watchdog = setTimeout(() => { - console.error('[Worker] Shutdown did not finish in 10s, exiting anyway'); + console.error( + `[Worker] Shutdown did not finish in ${SHUTDOWN_WATCHDOG_MS}ms, exiting anyway`, + ); process.exit(1); - }, 10_000); - watchdog.unref(); + }, SHUTDOWN_WATCHDOG_MS); try { + // Close the listener FIRST. Worker.stop() blocks until in-flight jobs + // finish (up to 300s), and advertising a healthy /health for the whole + // drain window tells the platform to keep routing to a replica that has + // already committed to dying. + healthServer.close(); scheduler?.stop(); await worker?.stop(); - healthServer.close(); await prisma.$disconnect(); console.log('[Worker] Shutdown complete'); process.exit(0); } catch (error) { - // These reject in practice: a SIGTERM during boot leaves $disconnect() - // tearing down a connection that was never established (P2024). Without - // this the rejection is unhandled and the process dies to a stack trace - // mid-shutdown instead of reporting a failed stop. + // Observed: signalled mid-boot, $disconnect() rejects while tearing down + // a pool that never filled ("Timed out fetching a new connection from the + // connection pool"). Without this the rejection is unhandled and the + // process dies to a stack trace mid-shutdown instead of reporting a + // failed stop. console.error('[Worker] Shutdown failed:', error); process.exit(1); } @@ -127,6 +197,28 @@ async function shutdown(signal: string): Promise { process.on('SIGTERM', () => void shutdown('SIGTERM')); process.on('SIGINT', () => void shutdown('SIGINT')); +// ─── Last-Resort Handlers ───────────────────────────────────────────────── + +// The whole design rests on this process staying up to explain itself, and under +// Node's defaults a single unhandled rejection ends it with a bare stack trace — +// back to the undiagnosable behaviour. Worker.poll() is fired unawaited from a +// timer and Worker's own signal handler calls stop() unawaited, so the paths +// exist. Mark the process unhealthy so /health tells the platform to stop routing +// to it, publish the reason, and then exit so the restart policy retries rather +// than leaving a wedged replica behind. +function failFatally(kind: string, error: unknown): void { + console.error(`[Worker] ${kind}:`, error); + if (boot.phase !== 'failed') { + boot = { phase: 'failed', error: `${kind}: ${summarizeBootError(error)}` }; + } + if (!shuttingDown) { + setTimeout(() => process.exit(1), 5_000); + } +} + +process.on('unhandledRejection', (reason) => failFatally('UNHANDLED REJECTION', reason)); +process.on('uncaughtException', (error) => failFatally('UNCAUGHT EXCEPTION', error)); + // ─── Boot ───────────────────────────────────────────────────────────────── // Everything that can throw at boot lives in here: buildSyncEngine's three @@ -170,13 +262,39 @@ async function startWorker(): Promise { started.on(JobType.JOB_CLEANUP, handleJobCleanup); started.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); + // A SIGTERM can land while buildSyncEngine() is still awaiting. shutdown() + // then runs to completion against null handles and heads for process.exit, + // and without this check the boot would resume behind it: Scheduler.start() + // ticks every definition immediately (enqueueing jobs) and Worker.start() + // begins claiming them, so the exit strands freshly-claimed rows in + // PROCESSING. Nothing sequences the two promise chains, so the flag is what + // sequences them. + if (shuttingDown) { + console.log('[Worker] Boot completed after shutdown began — not starting the worker'); + return; + } + // Published before start() so a probe landing mid-start sees the real worker, // and so shutdown can stop it if a signal arrives during boot. + const nextScheduler = new Scheduler(); worker = started; - scheduler = new Scheduler(); + scheduler = nextScheduler; - scheduler.start(); - started.start(); + try { + nextScheduler.start(); + started.start(); + } catch (error) { + // Scheduler.start() ticks every definition immediately and installs + // setInterval timers, so a throw between it and worker.start() would + // otherwise leave a process that reports itself failed while still + // enqueueing jobs nothing will consume. Torn down through the locals — + // the module-level handles are narrowed to null at the outer catch. + nextScheduler.stop(); + await started.stop().catch(() => {}); + worker = null; + scheduler = null; + throw error; + } } // NOTHING IS RETHROWN HERE, deliberately. This is a top-level-await entry @@ -191,19 +309,36 @@ async function startWorker(): Promise { // must never be reported healthy, because TRACKER_SYNC would write wrong // statuses to Linear. Railway fails the deploy on the failing healthcheck and // keeps the previous replica serving — same outcome, with a reason attached. +// How long a failed boot keeps answering 503 with its reason before exiting so +// restartPolicyType="ALWAYS" retries. Long enough for a deploy healthcheck and a +// log scrape to read it; short enough that a transient database outage recovers +// on its own rather than waiting for a human. +const BOOT_FAILURE_LINGER_MS = Number(process.env.BOOT_FAILURE_LINGER_MS ?? 120_000); + try { await startWorker(); - boot.phase = 'ready'; - console.log('[Worker] Worker process started'); + if (!shuttingDown) { + boot = { phase: 'ready' }; + console.log('[Worker] Worker process started'); + } } catch (error) { - boot.error = summarizeBootError(error); - boot.phase = 'failed'; + // startWorker() has already torn down anything it managed to start, so by + // here the process holds no timers and no poll loop — only the health server. + // // The full error goes to the logs only — /health carries the redacted form, - // since Prisma's connectivity errors quote the database host, port and user. + // since Prisma's errors quote the database host, port, user and container paths. + boot = { phase: 'failed', error: summarizeBootError(error) }; console.error('[Worker] BOOT FAILED:', error); console.error( - `[Worker] The process stays up so /health on ${port} reports 503 ("${boot.error}"). ` + + `[Worker] /health on ${port} reports 503 ("${boot.error}") for ${BOOT_FAILURE_LINGER_MS}ms, ` + + `then this process exits 1 so Railway's restart policy retries. ` + `A missing table or column here means the database does not match schema.prisma — ` + `check the schema-drift guard in apps/worker/start.sh.`, ); + setTimeout(() => { + console.error( + '[Worker] Exiting after the boot-failure linger window; restart policy takes over.', + ); + process.exit(1); + }, BOOT_FAILURE_LINGER_MS); } diff --git a/apps/worker/start.sh b/apps/worker/start.sh index fd523d61..c29a8259 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -4,8 +4,19 @@ set -e PRISMA="node /opt/prisma/node_modules/prisma/build/index.js" SCHEMA="packages/outpost/db/prisma/schema.prisma" -echo "Running database migrations..." -$PRISMA migrate deploy --schema "$SCHEMA" +echo "[worker] Running database migrations..." +# `migrate deploy` failing is far more common than drift and, left to bare `set -e`, +# dies with Prisma's output and no framing — in a script whose entire purpose is +# naming the real cause. Both web and worker migrate into the same database and +# their Railway logs interleave, hence the [worker] prefixes throughout. +if ! $PRISMA migrate deploy --schema "$SCHEMA"; then + echo "" + echo "[worker] FATAL: prisma migrate deploy failed (see the error above)." + echo "[worker] Common causes: a migration recorded as failed in _prisma_migrations (P3009)," + echo "[worker] a migration file edited after it was applied (P3006), or an unreachable database." + echo "[worker] The worker will not start against a database whose migrations did not apply." + exit 1 +fi # Schema-drift guard. # @@ -21,41 +32,67 @@ $PRISMA migrate deploy --schema "$SCHEMA" # 2026-08-07 failed with nothing but "1/1 replicas never became healthy" — nine # days of a five-minute healthcheck timeout that looked like a broken image. # -# `migrate diff` compares the LIVE DATABASE against schema.prisma and exits -# non-zero when they differ, which catches that class. Running it here means the -# deploy fails in seconds with the drifted object named, instead of timing out. +# ONLY ONE DIRECTION IS FATAL. `migrate diff` is bidirectional: it also emits DROP +# statements for anything in the live database that schema.prisma does not declare +# — a hand-added index, a leftover from a reverted feature, another tool's table, +# or (routinely) a migration a sibling service deployed before this image shipped. +# Failing on those would turn every normal schema-advancing rollout into a worker +# outage, because restartPolicyType="ALWAYS" would crash-loop this container until +# its own new image landed. Missing objects are the class that actually breaks the +# worker, so the guard fails on CREATE/ADD and merely warns on DROP. # # Deliberately fatal rather than a warning: the worker's sync mappings come from # the database, and one running against a schema it does not match would write # wrong statuses to Linear. Failing the deploy keeps the previous replica serving. -echo "Checking for schema drift..." +echo "[worker] Checking for schema drift..." set +e -$PRISMA migrate diff \ +DRIFT_SQL=$($PRISMA migrate diff \ --from-schema-datasource "$SCHEMA" \ --to-schema-datamodel "$SCHEMA" \ - --exit-code + --script 2>&1) DRIFT_STATUS=$? set -e -# --exit-code has three outcomes: 0 no difference, 2 a difference, and anything -# else the CLI itself failing (database unreachable, bad DATABASE_URL, schema -# engine did not start). Those are different problems and must not be reported -# with the same message — a script whose whole purpose is naming the real cause -# should not send the on-call hunting for drift that was never detected. -if [ "$DRIFT_STATUS" -eq 2 ]; then +if [ "$DRIFT_STATUS" -ne 0 ]; then echo "" - echo "FATAL: the database does not match schema.prisma (see the diff above)." - echo "A migration may be recorded as applied without having run." - echo "Compare models in schema.prisma against the live tables before redeploying." - exit 1 -elif [ "$DRIFT_STATUS" -ne 0 ]; then + echo "$DRIFT_SQL" echo "" - echo "FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." - echo "This is a tool or connectivity failure, NOT confirmed drift: the database" - echo "was never successfully compared. Check DATABASE_URL and that the database" - echo "is reachable from this container, then redeploy." + echo "[worker] FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." + echo "[worker] This is a tool or connectivity failure, NOT confirmed drift: the database" + echo "[worker] was never successfully compared. Check DATABASE_URL and that the database" + echo "[worker] is reachable from this container, then redeploy." exit 1 fi -echo "Migrations complete and schema matches. Starting worker..." +# Statements that CREATE or ADD are objects schema.prisma declares and the database +# lacks — the outage class. Anything else in the diff is an extra object the schema +# does not know about, which the worker does not care about. +MISSING=$(printf '%s\n' "$DRIFT_SQL" | grep -Ei '^[[:space:]]*(CREATE|ALTER[[:space:]]+TABLE.*[[:space:]]ADD[[:space:]])' || true) +EXTRA=$(printf '%s\n' "$DRIFT_SQL" | grep -Ei '^[[:space:]]*DROP' || true) + +if [ -n "$EXTRA" ]; then + echo "[worker] NOTE: the database contains objects schema.prisma does not declare." + echo "[worker] Not fatal — this is normal while a sibling service is mid-rollout." + printf '%s\n' "$EXTRA" | sed 's/^/[worker] /' +fi + +if [ -n "$MISSING" ]; then + echo "" + echo "[worker] Objects schema.prisma declares that the database does not have:" + printf '%s\n' "$MISSING" | sed 's/^/[worker] /' + echo "" + if [ "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" = "1" ]; then + echo "[worker] WARNING: OUTPOST_ALLOW_SCHEMA_DRIFT=1 is set — starting anyway." + echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." + echo "[worker] WARNING: unset this variable as soon as the schema is fixed." + else + echo "[worker] FATAL: the database is missing objects that schema.prisma declares." + echo "[worker] A migration may be recorded as applied without having run." + echo "[worker] Compare models in schema.prisma against the live tables before redeploying." + echo "[worker] To start anyway during an incident, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." + exit 1 + fi +fi + +echo "[worker] Migrations complete and schema matches. Starting worker..." exec node apps/worker/dist/index.js diff --git a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql index 0c0b9e57..fe7b8ce1 100644 --- a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql +++ b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql @@ -15,6 +15,13 @@ -- IF NOT EXISTS is deliberate: environments whose 0001_init did create the -- table must no-op rather than fail. Column definitions match the SystemConfig -- block in 0001_init exactly. +-- +-- It repairs exactly one state — table absent. A SystemConfig that exists with +-- the WRONG columns is not repaired: this no-ops, gets recorded as applied, and +-- leaves the same "recorded but not effective" gap it was written to close. That +-- state is caught at deploy time by the schema-drift guard in +-- apps/worker/start.sh, which reports the missing column rather than the missing +-- table; repairing it needs an ALTER, not this file. CREATE TABLE IF NOT EXISTS "SystemConfig" ( "key" TEXT NOT NULL, "value" TEXT NOT NULL, From d0c6f99f68fde738ba248beecedd19b0a8608159 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 10:27:31 -0700 Subject: [PATCH 75/83] fix(worker): reduce to the verified hotfix; defer the /health rework Two CR rounds (7 agents each) converged on the same verdict: the repair migration is correct and stable, and the boot//health rework is not ready. Round 1 found ~15 defects. Round 2 found ~20 more -- in round 1's fixes, not in the original code. Three of them would have caused incidents: - STALE_POLL_MS=60s answered 503 "stalled" for a HEALTHY worker. Worker.poll() stamps lastPollTime then awaits the job batch, and jobTimeouts allow 300s, so any long job froze the timestamp past the bound. The Docker healthcheck marks the container dead after 90s. A working worker would have been killed mid-job. The same commit had explicitly sized SHUTDOWN_WATCHDOG_MS *above* the 300s timeout for exactly this reason. - The drift guard's regex classifier silently passed real drift. Missing enum values (ALTER TYPE ... ADD VALUE) and wrong column types (ALTER COLUMN) match neither pattern, so the script printed "schema matches" against a drifted database -- the identical failure mode to `migrate deploy` reporting "no pending migrations", which is the bug this guard exists to compensate for. - Number(process.env.X ?? default) collapsed to 0 on a cleared platform variable, silently disabling the boot-failure linger window and forcing every SIGTERM to exit(1) mid-drain. Written one file away from a docblock explaining why ?? is insufficient for exactly this. The meta-pattern, as one reviewer put it: the module hardened its inputs in one place and trusted them everywhere else. That is a design that needs time and tests, not another round inside a hotfix. So this PR is reduced to what is verified: - The repair migration. Reviewed by every agent in both rounds; column definitions confirmed byte-for-byte against 0001_init and schema.prisma each time. This is what unblocks staging, whose worker last deployed 2026-07-24 and would otherwise hit the same missing table. - The schema-drift guard, in its plain `migrate diff --exit-code` form. ANY difference is fatal. The classifier that tried to be cleverer about sibling skew is gone; OUTPOST_ALLOW_SCHEMA_DRIFT (1/true/yes) is the release valve instead. A guard that can be wrong in the reassuring direction is worse than no guard. - Framing for a migrate deploy failure, [worker] log prefixes (web and worker interleave in Railway logs), and a startup signal trap so a SIGTERM during migration is not discarded by PID 1. - The Prisma CLI pinned to the lockfile's 6.19.3. It now gates whether the worker boots, and `prisma@6` floated -- resolving that range during review picked up 7.9.1, which rejects this schema outright. apps/worker/src/index.ts returns to main's behaviour: boot failures are fatal and opaque, exactly as they are in production today. That is a known quantity. Shipping a rework that answers 503 for healthy workers is not. The rework and its full finding list are preserved on wip/worker-health-boot-rework for a follow-up PR. Verified against a throwaway Postgres: clean DB starts; a missing table, a missing enum value, and a wrong column type each fail with exit 1 (the last two are the classes the deleted classifier passed); OUTPOST_ALLOW_SCHEMA_DRIFT=1, true and yes each start with warnings; an unreachable database reports a tool failure rather than confirmed drift. turbo typecheck 10/10, turbo test 10/10, sh -n clean. --- apps/worker/Dockerfile | 15 +- apps/worker/railway.toml | 6 - apps/worker/src/__tests__/health.test.ts | 280 -------------- apps/worker/src/health.ts | 213 ---------- apps/worker/src/index.ts | 363 ++++-------------- apps/worker/start.sh | 108 +++--- .../migration.sql | 24 +- 7 files changed, 151 insertions(+), 858 deletions(-) delete mode 100644 apps/worker/src/__tests__/health.test.ts delete mode 100644 apps/worker/src/health.ts diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index c432f0ce..f82cfd98 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -45,10 +45,11 @@ RUN chmod +x apps/worker/start.sh # Prisma CLI for running migrations at startup (installed standalone, mirrors apps/web/Dockerfile — # the pnpm workspace .bin paths don't resolve reliably in the pruned production image). -# Pinned to the lockfile's version, not the `prisma@6` range: start.sh now branches on -# this CLI's `migrate diff` output to decide whether the worker may boot at all, so a -# floating 6.x could change that contract and block every deploy. Keep in step with -# @prisma/client — a CLI/client mismatch can itself report false drift. +# Pinned to the lockfile's version rather than the `prisma@6` range: start.sh now gates +# the worker's boot on this CLI's `migrate diff` exit code, so a CLI change could block +# every deploy. Not hypothetical — resolving the floating range during review picked up +# 7.9.1, which rejects this schema outright. Keep in step with @prisma/client; a +# CLI/client mismatch can itself report false drift. RUN npm install --prefix /opt/prisma --no-save prisma@6.19.3 USER outpost @@ -56,11 +57,7 @@ ENV NODE_ENV=production ENV HEALTH_PORT=3005 EXPOSE 3005 -# Follows the same PORT-over-HEALTH_PORT precedence as src/index.ts. Hardcoding 3005 -# meant that whenever the platform injected PORT, the app moved and this probe failed -# forever against a healthy worker. 127.0.0.1 rather than localhost, which can resolve -# to ::1 inside the container and be refused (see apps/web/Dockerfile). HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD wget -qO- "http://127.0.0.1:${PORT:-${HEALTH_PORT:-3005}}/health" || exit 1 + CMD wget -qO- http://localhost:3005/health || exit 1 CMD ["./apps/worker/start.sh"] diff --git a/apps/worker/railway.toml b/apps/worker/railway.toml index 0dd66015..d99b38bf 100644 --- a/apps/worker/railway.toml +++ b/apps/worker/railway.toml @@ -9,9 +9,3 @@ dockerfilePath = "apps/worker/Dockerfile" [deploy] restartPolicyType = "ALWAYS" healthcheckPath = "/health" -# A failed boot now answers 503 with its reason immediately rather than dying -# silently, so there is nothing to gain from Railway's 300s default — that -# five-minute wait per attempt is what made the 2026-08-07 incident so slow to -# read. Kept above the worker's BOOT_FAILURE_LINGER_MS (120s) so the probe sees -# the reason before the process exits to be restarted. -healthcheckTimeout = 180 diff --git a/apps/worker/src/__tests__/health.test.ts b/apps/worker/src/__tests__/health.test.ts deleted file mode 100644 index 770eac60..00000000 --- a/apps/worker/src/__tests__/health.test.ts +++ /dev/null @@ -1,280 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; -import { - buildHealthResponse, - resolvePort, - summarizeBootError, - STALE_POLL_MS, - type BootState, -} from '../health.js'; - -const NOW = new Date('2026-08-12T22:00:00.000Z').getTime(); - -// Typed as the real contract, so a rename or removal in WorkerHealthStatus fails -// this file instead of leaving it green against a shape /health never serves. -const WORKER_HEALTH: WorkerHealthStatus = { - running: true, - activeJobCount: 2, - activeJobsByType: { AI_RESPONSE: 2 }, - lastPollTime: new Date(NOW - 1_000), - registeredHandlers: ['AI_RESPONSE', 'TRACKER_SYNC'], - upSince: new Date(NOW - 3_600_000), -}; - -describe('buildHealthResponse', () => { - it('reports 200 with the worker snapshot once boot is ready and the worker is polling', () => { - const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); - - expect(statusCode).toBe(200); - expect(body).toMatchObject({ status: 'ok', running: true, activeJobCount: 2 }); - }); - - // `status` is the envelope's field. Spreading the snapshot over it would let a - // future WorkerHealthStatus.status redefine "ok" for every probe silently. - it('keeps its own status field even if the snapshot carries one', () => { - const shadowed = { ...WORKER_HEALTH, status: 'degraded' } as unknown as WorkerHealthStatus; - - const { body } = buildHealthResponse({ phase: 'ready' }, shadowed, NOW); - - expect(body.status).toBe('ok'); - }); - - // The regression this pins: the worker used to await the database at module - // scope, above the health server, so a boot failure exited the process before - // anything bound the port. Railway could only say "1/1 replicas never became - // healthy" — indistinguishable from a broken image, and it hid a missing - // SystemConfig table for nine days. A failed boot must now answer, and the - // answer must carry the reason. - it('reports 503 AND the reason when boot failed', () => { - const boot: BootState = { - phase: 'failed', - error: 'P2021: missing database object `public.SystemConfig`', - }; - - const { statusCode, body } = buildHealthResponse(boot, null, NOW); - - expect(statusCode).toBe(503); - expect(body.status).toBe('failed'); - expect(body.error).toContain('SystemConfig'); - }); - - it('reports 503 with a reason while boot is still in progress', () => { - const { statusCode, body } = buildHealthResponse({ phase: 'starting' }, null, NOW); - - expect(statusCode).toBe(503); - expect(body.status).toBe('starting'); - expect(body.error).toBeTruthy(); - }); - - // Fail-fast is retained on purpose: a worker whose sync mappings could not be - // read must never be routed to, because it would write wrong statuses to - // Linear. This pins that a failed boot is not quietly downgraded to healthy. - it('never returns 200 for a failed boot, even with a worker snapshot present', () => { - const boot: BootState = { phase: 'failed', error: 'connection refused' }; - - const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH, NOW); - - expect(statusCode).toBe(503); - expect(body.error).toBe('connection refused'); - }); - - // The gap that actually reaches production. Worker.stop() sets running=false - // without exiting the process — including from Worker's OWN signal handlers — - // so gating the 200 on the snapshot's existence alone answered - // 200 {"status":"ok","running":false} for a worker processing nothing. - it('does not report 200 for a worker that has stopped', () => { - const stopped: WorkerHealthStatus = { ...WORKER_HEALTH, running: false, upSince: null }; - - const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stopped, NOW); - - expect(statusCode).toBe(503); - expect(body.status).toBe('stopped'); - expect(body.error).toContain('not running'); - }); - - // Worker.poll() catches every error and reschedules, so a poll blocked on a - // hung database call leaves running=true forever with lastPollTime frozen. - it('does not report 200 for a worker whose poll loop has stalled', () => { - const stalled: WorkerHealthStatus = { - ...WORKER_HEALTH, - lastPollTime: new Date(NOW - STALE_POLL_MS - 1), - }; - - const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stalled, NOW); - - expect(statusCode).toBe(503); - expect(body.status).toBe('stalled'); - }); - - it('does not report 200 for a worker that has never polled', () => { - const neverPolled: WorkerHealthStatus = { ...WORKER_HEALTH, lastPollTime: null }; - - const { statusCode } = buildHealthResponse({ phase: 'ready' }, neverPolled, NOW); - - expect(statusCode).toBe(503); - }); - - // A half-booted worker must not be reported healthy just because the phase - // flag says ready — and the 503 must still explain itself rather than - // answering {"status":"ready","error":null}, which is the reasonless body - // this endpoint exists to eliminate. - it('reports 503 with a reason when the phase is ready but no worker exists', () => { - const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, null, NOW); - - expect(statusCode).toBe(503); - expect(body.status).toBe('no-worker'); - expect(body.error).toContain('boot sequence'); - }); - - it('serves a body that survives JSON serialization', () => { - const { body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); - - expect(() => JSON.stringify(body)).not.toThrow(); - expect(JSON.parse(JSON.stringify(body))).toMatchObject({ status: 'ok', running: true }); - }); -}); - -// /health is unauthenticated, so whatever lands in boot.error is published. -// Prisma's connectivity errors quote the database host, port and user; its -// schema errors arrive as a multi-line blob whose preamble carries an absolute -// container path and a source code frame. Only the object name may survive. -describe('summarizeBootError', () => { - // The shape Prisma actually throws — not a hand-built single-line message. - const realisticP2021 = Object.assign( - new Error( - 'Invalid `prisma.systemConfig.findUnique()` invocation in\n' + - '/app/packages/outpost/shared/dist/sync/config.js:34:56\n\n' + - ' 31 const existing = await db.systemConfig.findUnique({\n\n' + - 'The table `public.SystemConfig` does not exist in the current database.', - ), - { code: 'P2021' }, - ); - - it('names the missing object without leaking container paths or the code frame', () => { - const summary = summarizeBootError(realisticP2021); - - expect(summary).toContain('P2021'); - expect(summary).toContain('public.SystemConfig'); - expect(summary).not.toContain('/app/'); - expect(summary).not.toContain('findUnique'); - }); - - it('covers P2022 missing-column drift, not just P2021', () => { - const error = Object.assign( - new Error( - 'The column `public.SystemConfig.updatedAt` does not exist in the current database.', - ), - { code: 'P2022' }, - ); - - expect(summarizeBootError(error)).toContain('P2022'); - expect(summarizeBootError(error)).toContain('SystemConfig.updatedAt'); - }); - - it('redacts the host and user out of a connectivity error', () => { - const error = Object.assign( - new Error("Can't reach database server at `db.internal.railway.app:5432`"), - { code: 'P1001' }, - ); - - const summary = summarizeBootError(error); - - expect(summary).toContain('P1001'); - expect(summary).not.toContain('db.internal.railway.app'); - expect(summary).not.toContain('5432'); - }); - - // PrismaClientInitializationError carries `errorCode`, not `code` — the real - // shape observed from a live boot against an unreachable database. - it('reads errorCode as well as code', () => { - const error = Object.assign( - new Error('Timed out fetching a new connection from the pool'), - { - errorCode: 'P2024', - }, - ); - - expect(summarizeBootError(error)).toContain('P2024'); - }); - - it('falls back to the error class when no code is present at all', () => { - const error = new Error("Can't reach database server at `127.0.0.1:59999`"); - error.name = 'PrismaClientInitializationError'; - - const summary = summarizeBootError(error); - - expect(summary).toContain('PrismaClientInitializationError'); - expect(summary).not.toContain('127.0.0.1'); - }); - - it('redacts credentials out of a plain error', () => { - const summary = summarizeBootError(new Error('postgres://user:hunter2@host/db refused')); - - expect(summary).not.toContain('hunter2'); - }); - - it('handles thrown non-Error values', () => { - expect(summarizeBootError('boom')).toBeTruthy(); - expect(summarizeBootError(null)).toBeTruthy(); - expect(summarizeBootError(undefined)).toBeTruthy(); - }); - - // A plain object carrying a safe code is the one case the echo branch exists - // for; String(obj) would render "[object Object]". - it('reads .message off a non-Error object carrying a safe code', () => { - const summary = summarizeBootError({ - code: 'P2021', - message: 'The table `public.SystemConfig` does not exist in the current database.', - }); - - expect(summary).toContain('public.SystemConfig'); - expect(summary).not.toContain('[object Object]'); - }); - - it('bounds the length of anything it serves', () => { - const error = Object.assign( - new Error(`The table \`${'x'.repeat(5_000)}\` does not exist.`), - { - code: 'P2021', - }, - ); - - expect(summarizeBootError(error).length).toBeLessThanOrEqual(200); - }); -}); - -// An invalid port used to reach server.listen() as NaN, which throws -// ERR_SOCKET_BAD_PORT synchronously at module scope — killing the process before -// anything bound, the exact opaque failure the health server exists to prevent. -describe('resolvePort', () => { - it('prefers PORT, then HEALTH_PORT, then the default', () => { - expect(resolvePort({ PORT: '8080', HEALTH_PORT: '3005' })).toMatchObject({ - port: 8080, - source: 'PORT', - }); - expect(resolvePort({ HEALTH_PORT: '3005' })).toMatchObject({ - port: 3005, - source: 'HEALTH_PORT', - }); - expect(resolvePort({})).toMatchObject({ port: 3003, source: 'default' }); - }); - - // The reported trigger: `??` only falls through on null/undefined, so a - // cleared platform variable arrives as '' and parses to NaN. - it('falls back with a warning on an empty PORT rather than yielding NaN', () => { - const resolved = resolvePort({ PORT: '', HEALTH_PORT: '3005' }); - - expect(resolved.port).toBe(3005); - expect(resolved.source).toBe('HEALTH_PORT'); - }); - - it('falls back with a warning on a non-numeric or out-of-range PORT', () => { - for (const bad of ['tcp://host:5432', 'abc', '70000', '-1']) { - const resolved = resolvePort({ PORT: bad }); - - expect(resolved.port).toBe(3003); - expect(resolved.warning).toContain(bad); - expect(Number.isInteger(resolved.port)).toBe(true); - } - }); -}); diff --git a/apps/worker/src/health.ts b/apps/worker/src/health.ts deleted file mode 100644 index 7834c4d4..00000000 --- a/apps/worker/src/health.ts +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Boot state, health payload construction, and port resolution — split out from - * index.ts so they are testable. - * - * index.ts is a top-level-await module with side effects on import (it binds a - * port and starts polling), so its boot behaviour cannot be exercised directly - * from a test. Everything here is pure and pinned by tests: an unbooted, failed, - * stopped or stalled worker must report 503 WITH a reason, and only a worker - * that is actually polling reports 200. - */ - -import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; - -export type BootPhase = 'starting' | 'ready' | 'failed'; - -/** - * Discriminated so the failed-without-a-reason state is unrepresentable. A 503 - * carrying `error: null` is the exact signal-quality bug this module exists to - * remove; making it a type error is cheaper than remembering to assign `error` - * before `phase` on every future edit. - */ -export type BootState = - | { phase: 'starting' } - | { phase: 'ready' } - | { phase: 'failed'; error: string }; - -export interface HealthResponse { - statusCode: number; - body: Record; -} - -/** - * Prisma error codes whose failure names a schema object rather than a - * connection. P2021 is a missing table, P2022 a missing column — the drift class - * this endpoint exists to surface. Only the object name is echoed, never the - * message (see summarizeBootError). - */ -const SAFE_TO_ECHO_CODES = new Set(['P2021', 'P2022']); - -/** Worker.poll() reschedules every 1s, so a minute of silence means it is wedged. */ -export const STALE_POLL_MS = 60_000; - -/** Upper bound on any reason string served to an unauthenticated probe. */ -const MAX_REASON_LENGTH = 200; - -function messageOf(error: unknown): string { - if (error instanceof Error) return error.message; - if ( - typeof error === 'object' && - error !== null && - typeof (error as { message?: unknown }).message === 'string' - ) { - return (error as { message: string }).message; - } - return String(error); -} - -/** - * Reduce a boot exception to a reason that can be served on /health. - * - * /health is unauthenticated. Prisma's connectivity errors quote the database - * host, port and user (P1001 names host:port, P1000 names the user), and even - * the "safe" schema errors arrive as a multi-line blob whose preamble carries - * an absolute container path and a source code frame: - * - * Invalid `prisma.systemConfig.findUnique()` invocation in - * /app/packages/outpost/shared/dist/sync/config.js:34:56 - * 31 const existing = await db.systemConfig.findUnique({ - * The table `public.SystemConfig` does not exist in the current database. - * - * So nothing is echoed verbatim. For the schema codes the backticked object name - * is lifted out of the final line — that name is the entire diagnostic payload — - * and everything else degrades to error class plus code, with the full text left - * to the logs. - */ -export function summarizeBootError(error: unknown): string { - const raw = (error ?? {}) as { code?: unknown; errorCode?: unknown }; - // PrismaClientKnownRequestError carries `code`; PrismaClientInitializationError - // carries `errorCode` (frequently undefined, hence the class-name fallback). - const code = - typeof raw.code === 'string' - ? raw.code - : typeof raw.errorCode === 'string' - ? raw.errorCode - : null; - - if (code && SAFE_TO_ECHO_CODES.has(code)) { - const lastLine = messageOf(error) - .split('\n') - .map((line) => line.trim()) - .filter(Boolean) - .at(-1); - const object = lastLine ? /`([^`]+)`/.exec(lastLine)?.[1] : undefined; - if (object) { - return truncate( - `${code}: missing database object \`${object}\` — the database does not match schema.prisma`, - ); - } - } - - // Class name and code only. Both are stable, neither quotes the connection. - const label = [error instanceof Error ? error.name : 'Error', code].filter(Boolean).join(' '); - return truncate(`${label} — see the worker logs for the full error`); -} - -function truncate(reason: string): string { - return reason.length <= MAX_REASON_LENGTH - ? reason - : `${reason.slice(0, MAX_REASON_LENGTH - 1)}…`; -} - -/** - * Resolve the health-server port from the environment. - * - * `??` is not enough: an empty or non-numeric PORT (a cleared platform variable, - * or a reference variable that failed to resolve) parses to NaN, and - * `server.listen(NaN)` throws ERR_SOCKET_BAD_PORT synchronously at module scope - * — killing the process before anything binds, which is precisely the opaque - * "replicas never became healthy" failure this whole module exists to prevent. - * An invalid value falls back to the default and says so. - */ -export function resolvePort( - env: { PORT?: string; HEALTH_PORT?: string }, - fallback = 3003, -): { port: number; source: string; warning: string | null } { - const candidates: Array<[string, string | undefined]> = [ - ['PORT', env.PORT], - ['HEALTH_PORT', env.HEALTH_PORT], - ]; - - for (const [source, raw] of candidates) { - if (raw === undefined || raw.trim() === '') continue; - const parsed = Number.parseInt(raw, 10); - if (Number.isInteger(parsed) && parsed >= 0 && parsed <= 65535) { - return { port: parsed, source, warning: null }; - } - return { - port: fallback, - source: 'default', - warning: `invalid ${source}="${raw}" (expected an integer 0-65535), falling back to ${fallback}`, - }; - } - - return { port: fallback, source: 'default', warning: null }; -} - -/** - * Build the /health response. - * - * `workerHealth` is the worker's own snapshot, or null when the worker has not - * been constructed. It is passed rather than read so this stays pure. - * - * Every non-200 answer carries a reason. The value added over simply exiting is - * that body: a probe alone explains the failure. Exiting before binding the port - * is what made a missing SystemConfig table look identical to a broken image for - * nine days. - * - * 200 requires the worker to be *polling*, not merely constructed. Worker.stop() - * sets running=false without exiting the process, and a poll blocked on a hung - * database call freezes lastPollTime — both leave a worker that processes - * nothing while looking alive, which is the honesty gap tracked by #138. - */ -export function buildHealthResponse( - boot: BootState, - workerHealth: WorkerHealthStatus | null, - now: number = Date.now(), -): HealthResponse { - if (boot.phase === 'failed') { - return { statusCode: 503, body: { status: 'failed', error: boot.error } }; - } - - if (boot.phase === 'starting') { - return { statusCode: 503, body: { status: 'starting', error: 'boot has not finished' } }; - } - - if (!workerHealth) { - return { - statusCode: 503, - body: { - status: 'no-worker', - error: 'boot reported ready but no worker was constructed — this is a bug in the boot sequence, not a database problem', - }, - }; - } - - if (!workerHealth.running) { - return { - statusCode: 503, - body: { - ...workerHealth, - status: 'stopped', - error: 'worker is not running — it was stopped without the process exiting', - }, - }; - } - - const sincePoll = workerHealth.lastPollTime ? now - workerHealth.lastPollTime.getTime() : null; - if (sincePoll === null || sincePoll > STALE_POLL_MS) { - return { - statusCode: 503, - body: { - ...workerHealth, - status: 'stalled', - error: `worker has not polled for ${sincePoll ?? 'any'}ms — the poll loop is blocked, most likely on a hung database call`, - }, - }; - } - - // `status` last on purpose: it is the envelope's own field, and spreading the - // snapshot over it would let a future WorkerHealthStatus.status silently - // redefine what "ok" means to every probe. - return { statusCode: 200, body: { ...workerHealth, status: 'ok' } }; -} diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index eb532317..13d2b9a3 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -15,14 +15,6 @@ * - TRACKER_SYNC: Push changes to external trackers * - JOB_CLEANUP: Periodic cleanup of old jobs and sync events * - GITHUB_REACTION_POLL: Poll GitHub reactions on AI comments (no webhook exists) - * - * BOOT ORDER: /health starts listening before any database QUERY runs, so a boot - * failure is reported rather than merely fatal. See the boot-state block below. - * Two classes still escape it, both by construction: the `prisma` import below - * constructs a PrismaClient at module scope (it throws for an ungenerated client - * or an unparseable DATABASE_URL), and a failure to bind the port itself cannot - * be reported over the port. Both are handled loudly rather than silently — see - * the health-server error handler and the last-resort handlers at the bottom. */ import http from 'node:http'; @@ -42,303 +34,102 @@ import { handleGithubReactionPoll, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; -import { buildHealthResponse, resolvePort, summarizeBootError, type BootState } from './health.js'; -// ─── Boot state ─────────────────────────────────────────────────────────── +// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── -// Fail-fast on a bad boot is still the intent: a worker running with silently -// defaulted sync mappings would write wrong statuses to Linear, so it must not -// report itself healthy. What changed is that failing is no longer SILENT. -// -// This used to be a top-level `await buildSyncEngine()` above the health server, -// so any boot-time database problem killed the process before anything bound the -// port. Railway could only report "1/1 replicas never became healthy", which is -// indistinguishable from a broken image. That cost nine days of undiagnosed -// deploy failures when SystemConfig turned out to be missing from the production -// database: every deploy from 2026-08-07 failed with no usable signal. -// -// Now the port binds first and /health answers 503 with the reason while the boot -// is unfinished or failed, so the reason is one probe away instead of buried in -// container logs nobody had reason to suspect. +// BOOT SEMANTICS — deliberate change. This is a top-level await that performs +// three database reads (the persisted status / priority / label mapping configs) +// before this module finishes evaluating. If the database is unreachable at boot +// the import throws, so the process exits BEFORE the health server below starts +// listening: the container crash-loops with no /health at all rather than coming +// up and reporting itself degraded. // -// A failed boot does NOT park here forever. Railway's healthcheckPath gates a NEW -// DEPLOYMENT; it does not continuously probe and restart an already-running -// service, and restartPolicyType="ALWAYS" is a restart-on-exit policy that can -// never fire on a process that never exits. Staying up indefinitely would mean a -// 20-second Postgres failover during an ordinary container restart wedges the -// worker with zero jobs processed until a human notices — strictly worse than the -// crash-loop it replaced. So the reason is published for BOOT_FAILURE_LINGER_MS -// (long enough for the deploy probe and any log scrape to read it) and then the -// process exits non-zero so the restart policy retries. Diagnosable AND -// self-healing; the two were never actually in tension. -let boot: BootState = { phase: 'starting' }; +// Fail-fast is the intent — a worker running with silently-defaulted mappings is +// worse than one that is visibly down, since TRACKER_SYNC would then write wrong +// statuses to Linear. Railway's restart policy is the retry mechanism. Note this +// interacts with the /health honesty follow-up (#138): once /health reflects +// worker state, a degraded-but-listening mode becomes a real option and this +// decision is worth revisiting. +const syncEngine = await buildSyncEngine(); + +const handleTrackerSync = createTrackerSyncHandler(syncEngine); + +// ─── Create Worker ──────────────────────────────────────────────────────── + +const worker = new Worker({ + maxConcurrency: 10, + pollIntervalMs: 1000, + concurrencyByType: { + [JobType.AI_RESPONSE]: 4, + [JobType.ESCALATION]: 2, + [JobType.SLA_CHECK]: 1, + [JobType.ONBOARDING_DIGEST]: 1, + [JobType.ACCOUNT_SCORING]: 1, + [JobType.HUBSPOT_SYNC]: 1, + [JobType.TRACKER_SYNC]: 1, + [JobType.JOB_CLEANUP]: 1, + [JobType.GITHUB_REACTION_POLL]: 1, + }, + jobTimeouts: { + [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow + [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large + [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts + }, +}); + +// ─── Register Handlers ──────────────────────────────────────────────────── -let worker: Worker | null = null; -let scheduler: Scheduler | null = null; +worker.on(JobType.AI_RESPONSE, handleAiResponse); +worker.on(JobType.ESCALATION, handleEscalation); +worker.on(JobType.SLA_CHECK, handleSlaCheck); +worker.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); +worker.on(JobType.ACCOUNT_SCORING, handleAccountScoring); +worker.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); +worker.on(JobType.TRACKER_SYNC, handleTrackerSync); +worker.on(JobType.JOB_CLEANUP, handleJobCleanup); +worker.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); + +// ─── Start Scheduler ────────────────────────────────────────────────────── + +const scheduler = new Scheduler(); // ─── Health Server ──────────────────────────────────────────────────────── -const { port, source: portSource, warning: portWarning } = resolvePort(process.env); -if (portWarning) console.error(`[Worker] ${portWarning}`); +const port = parseInt(process.env.PORT ?? process.env.HEALTH_PORT ?? '3003', 10); const healthServer = http.createServer((req, res) => { - // A probe must never be able to kill the process it exists to observe: - // worker.healthCheck() and JSON.stringify both run in this callback, and an - // exception in an http listener is an uncaught exception. - try { - const path = - new URL(req.url ?? '/', 'http://localhost').pathname.replace(/\/+$/, '') || '/'; - if (path !== '/health') { - res.writeHead(404, { 'Content-Type': 'text/plain' }); - res.end('Not Found'); - return; - } - - const { statusCode, body } = buildHealthResponse( - boot, - worker ? worker.healthCheck() : null, - ); - res.writeHead(statusCode, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(body)); - } catch (error) { - console.error('[Worker] /health handler threw:', error); - res.writeHead(500, { 'Content-Type': 'application/json' }); - res.end( - JSON.stringify({ - status: 'error', - error: 'health handler failed — see the worker logs', - }), - ); + if (req.url === '/health') { + const health = worker.healthCheck(); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ status: 'ok', ...health })); + } else { + res.writeHead(404); + res.end('Not Found'); } }); -// listen() reports bind failures asynchronously through 'error'. With no listener -// that is an uncaught exception: the port never binds and the process dies to a -// bare stack trace — the same opaque signal as the original incident, arriving -// through the one step everything else now depends on. It is also the single -// failure that genuinely cannot be reported over /health, so it must be loud in -// the logs and must exit rather than linger pretending to serve. -healthServer.on('error', (error: NodeJS.ErrnoException) => { - console.error( - `[Worker] FATAL: could not bind the health server to port ${port} (${error.code ?? 'unknown'}, from ${portSource}). ` + - `Nothing can report this process's state without it. Check PORT/HEALTH_PORT and whether another process holds the port.`, - error, - ); - process.exit(1); -}); +// ─── Start Everything ───────────────────────────────────────────────────── healthServer.listen(port, () => { - console.log( - `[Worker] Health server listening on port ${port} (from ${portSource}, boot: ${boot.phase})`, - ); + console.log(`[Worker] Health server listening on port ${port}`); }); -// ─── Graceful Shutdown ──────────────────────────────────────────────────── +scheduler.start(); +worker.start(); -// Registered BEFORE the boot await, not after it. Boot is the slowest thing this -// process does, which is exactly when Railway tears a bad deploy down — and a -// SIGTERM arriving while module evaluation is still suspended would find no -// handler and kill the process outright. -// -// Note there is a SECOND registrar: Worker.start() installs its own SIGTERM / -// SIGINT handlers that call worker.stop() unawaited. Both fire. That is safe -// only because Worker.stop() early-returns on !running and this handler runs -// first, so ordering here is load-bearing — do not move this registration below -// startWorker(). -let shuttingDown = false; +console.log('[Worker] Worker process started'); -// Longer than the largest entry in jobTimeouts below (300s), because -// Worker.stop() waits for in-flight jobs to finish. A watchdog shorter than the -// drain it guards would turn every deploy that lands mid-job into a forced -// exit(1) — guarding the hang while breaking the normal path. -const SHUTDOWN_WATCHDOG_MS = Number(process.env.SHUTDOWN_WATCHDOG_MS ?? 330_000); +// ─── Graceful Shutdown ──────────────────────────────────────────────────── async function shutdown(signal: string): Promise { - if (shuttingDown) return; - shuttingDown = true; - console.log(`[Worker] Received ${signal} in boot phase '${boot.phase}', shutting down...`); - - // NOT unref'd. The motivating case is a $disconnect() that never settles - // after the server is closed — precisely when no other referenced handle - // remains, so an unref'd timer would let Node exit 0 (reporting a clean stop - // for a shutdown that never completed) and this line would never print. - // Every path below ends in process.exit, so a referenced timer costs nothing. - const watchdog = setTimeout(() => { - console.error( - `[Worker] Shutdown did not finish in ${SHUTDOWN_WATCHDOG_MS}ms, exiting anyway`, - ); - process.exit(1); - }, SHUTDOWN_WATCHDOG_MS); - - try { - // Close the listener FIRST. Worker.stop() blocks until in-flight jobs - // finish (up to 300s), and advertising a healthy /health for the whole - // drain window tells the platform to keep routing to a replica that has - // already committed to dying. - healthServer.close(); - scheduler?.stop(); - await worker?.stop(); - await prisma.$disconnect(); - console.log('[Worker] Shutdown complete'); - process.exit(0); - } catch (error) { - // Observed: signalled mid-boot, $disconnect() rejects while tearing down - // a pool that never filled ("Timed out fetching a new connection from the - // connection pool"). Without this the rejection is unhandled and the - // process dies to a stack trace mid-shutdown instead of reporting a - // failed stop. - console.error('[Worker] Shutdown failed:', error); - process.exit(1); - } -} - -// `void` because an unhandled rejection here would be the very failure the catch -// above exists to prevent. -process.on('SIGTERM', () => void shutdown('SIGTERM')); -process.on('SIGINT', () => void shutdown('SIGINT')); - -// ─── Last-Resort Handlers ───────────────────────────────────────────────── - -// The whole design rests on this process staying up to explain itself, and under -// Node's defaults a single unhandled rejection ends it with a bare stack trace — -// back to the undiagnosable behaviour. Worker.poll() is fired unawaited from a -// timer and Worker's own signal handler calls stop() unawaited, so the paths -// exist. Mark the process unhealthy so /health tells the platform to stop routing -// to it, publish the reason, and then exit so the restart policy retries rather -// than leaving a wedged replica behind. -function failFatally(kind: string, error: unknown): void { - console.error(`[Worker] ${kind}:`, error); - if (boot.phase !== 'failed') { - boot = { phase: 'failed', error: `${kind}: ${summarizeBootError(error)}` }; - } - if (!shuttingDown) { - setTimeout(() => process.exit(1), 5_000); - } -} - -process.on('unhandledRejection', (reason) => failFatally('UNHANDLED REJECTION', reason)); -process.on('uncaughtException', (error) => failFatally('UNCAUGHT EXCEPTION', error)); - -// ─── Boot ───────────────────────────────────────────────────────────────── - -// Everything that can throw at boot lives in here: buildSyncEngine's three -// database reads (the persisted status / priority / label mapping configs), the -// Worker construction, and the scheduler/worker start. Anything that escapes -// leaves boot.phase === 'failed' and the process ALIVE but unhealthy, so the -// reason reaches /health instead of vanishing with the process. -async function startWorker(): Promise { - const syncEngine = await buildSyncEngine(); - const handleTrackerSync = createTrackerSyncHandler(syncEngine); - - const started = new Worker({ - maxConcurrency: 10, - pollIntervalMs: 1000, - concurrencyByType: { - [JobType.AI_RESPONSE]: 4, - [JobType.ESCALATION]: 2, - [JobType.SLA_CHECK]: 1, - [JobType.ONBOARDING_DIGEST]: 1, - [JobType.ACCOUNT_SCORING]: 1, - [JobType.HUBSPOT_SYNC]: 1, - [JobType.TRACKER_SYNC]: 1, - [JobType.JOB_CLEANUP]: 1, - [JobType.GITHUB_REACTION_POLL]: 1, - }, - jobTimeouts: { - [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow - [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large - [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts - }, - }); - - // ─── Register Handlers ──────────────────────────────────────────────── - started.on(JobType.AI_RESPONSE, handleAiResponse); - started.on(JobType.ESCALATION, handleEscalation); - started.on(JobType.SLA_CHECK, handleSlaCheck); - started.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); - started.on(JobType.ACCOUNT_SCORING, handleAccountScoring); - started.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); - started.on(JobType.TRACKER_SYNC, handleTrackerSync); - started.on(JobType.JOB_CLEANUP, handleJobCleanup); - started.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); - - // A SIGTERM can land while buildSyncEngine() is still awaiting. shutdown() - // then runs to completion against null handles and heads for process.exit, - // and without this check the boot would resume behind it: Scheduler.start() - // ticks every definition immediately (enqueueing jobs) and Worker.start() - // begins claiming them, so the exit strands freshly-claimed rows in - // PROCESSING. Nothing sequences the two promise chains, so the flag is what - // sequences them. - if (shuttingDown) { - console.log('[Worker] Boot completed after shutdown began — not starting the worker'); - return; - } - - // Published before start() so a probe landing mid-start sees the real worker, - // and so shutdown can stop it if a signal arrives during boot. - const nextScheduler = new Scheduler(); - worker = started; - scheduler = nextScheduler; - - try { - nextScheduler.start(); - started.start(); - } catch (error) { - // Scheduler.start() ticks every definition immediately and installs - // setInterval timers, so a throw between it and worker.start() would - // otherwise leave a process that reports itself failed while still - // enqueueing jobs nothing will consume. Torn down through the locals — - // the module-level handles are narrowed to null at the outer catch. - nextScheduler.stop(); - await started.stop().catch(() => {}); - worker = null; - scheduler = null; - throw error; - } + console.log(`[Worker] Received ${signal}, shutting down...`); + scheduler.stop(); + await worker.stop(); + healthServer.close(); + await prisma.$disconnect(); + console.log('[Worker] Shutdown complete'); + process.exit(0); } -// NOTHING IS RETHROWN HERE, deliberately. This is a top-level-await entry -// module: an exception escaping module evaluation rejects its evaluation -// promise, which Node reports as an uncaught exception and exits on — a -// listening HTTP server does not keep the process alive. Rethrowing would kill -// the health server before it could answer a single probe and hand Railway the -// same bare "1/1 replicas never became healthy" that hid a missing SystemConfig -// table for nine days. Staying up and answering 503 IS the fix. -// -// Fail-fast is still the intent: a worker whose sync mappings could not be read -// must never be reported healthy, because TRACKER_SYNC would write wrong -// statuses to Linear. Railway fails the deploy on the failing healthcheck and -// keeps the previous replica serving — same outcome, with a reason attached. -// How long a failed boot keeps answering 503 with its reason before exiting so -// restartPolicyType="ALWAYS" retries. Long enough for a deploy healthcheck and a -// log scrape to read it; short enough that a transient database outage recovers -// on its own rather than waiting for a human. -const BOOT_FAILURE_LINGER_MS = Number(process.env.BOOT_FAILURE_LINGER_MS ?? 120_000); - -try { - await startWorker(); - if (!shuttingDown) { - boot = { phase: 'ready' }; - console.log('[Worker] Worker process started'); - } -} catch (error) { - // startWorker() has already torn down anything it managed to start, so by - // here the process holds no timers and no poll loop — only the health server. - // - // The full error goes to the logs only — /health carries the redacted form, - // since Prisma's errors quote the database host, port, user and container paths. - boot = { phase: 'failed', error: summarizeBootError(error) }; - console.error('[Worker] BOOT FAILED:', error); - console.error( - `[Worker] /health on ${port} reports 503 ("${boot.error}") for ${BOOT_FAILURE_LINGER_MS}ms, ` + - `then this process exits 1 so Railway's restart policy retries. ` + - `A missing table or column here means the database does not match schema.prisma — ` + - `check the schema-drift guard in apps/worker/start.sh.`, - ); - setTimeout(() => { - console.error( - '[Worker] Exiting after the boot-failure linger window; restart policy takes over.', - ); - process.exit(1); - }, BOOT_FAILURE_LINGER_MS); -} +process.on('SIGTERM', () => shutdown('SIGTERM')); +process.on('SIGINT', () => shutdown('SIGINT')); diff --git a/apps/worker/start.sh b/apps/worker/start.sh index c29a8259..edfac446 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -1,14 +1,21 @@ #!/bin/sh set -e +# Until `exec node` below, PID 1 is this shell. Linux discards signals that PID 1 +# has no handler for, so without this a SIGTERM arriving during migrate deploy is +# ignored outright and the platform waits out the full grace period before +# SIGKILLing — potentially mid-migration. +trap 'echo "[worker] received SIGTERM during startup, aborting"; exit 143' TERM +trap 'echo "[worker] received SIGINT during startup, aborting"; exit 130' INT + PRISMA="node /opt/prisma/node_modules/prisma/build/index.js" SCHEMA="packages/outpost/db/prisma/schema.prisma" echo "[worker] Running database migrations..." -# `migrate deploy` failing is far more common than drift and, left to bare `set -e`, -# dies with Prisma's output and no framing — in a script whose entire purpose is -# naming the real cause. Both web and worker migrate into the same database and -# their Railway logs interleave, hence the [worker] prefixes throughout. +# A migrate deploy failure is far more common than drift and, left to bare `set -e`, +# dies with Prisma's output and no framing — in a script whose whole purpose is +# naming the real cause. web and worker migrate into the same database and their +# logs interleave, hence the [worker] prefixes throughout. if ! $PRISMA migrate deploy --schema "$SCHEMA"; then echo "" echo "[worker] FATAL: prisma migrate deploy failed (see the error above)." @@ -27,35 +34,62 @@ fi # the tables that migration declares. # # That is exactly what happened: production had 0001_init recorded as applied -# while the SystemConfig table it declares did not exist. The worker read that -# table during boot, threw, and died before binding /health, so every deploy from +# while the SystemConfig table it declares did not exist. Every deploy from # 2026-08-07 failed with nothing but "1/1 replicas never became healthy" — nine # days of a five-minute healthcheck timeout that looked like a broken image. # -# ONLY ONE DIRECTION IS FATAL. `migrate diff` is bidirectional: it also emits DROP -# statements for anything in the live database that schema.prisma does not declare -# — a hand-added index, a leftover from a reverted feature, another tool's table, -# or (routinely) a migration a sibling service deployed before this image shipped. -# Failing on those would turn every normal schema-advancing rollout into a worker -# outage, because restartPolicyType="ALWAYS" would crash-loop this container until -# its own new image landed. Missing objects are the class that actually breaks the -# worker, so the guard fails on CREATE/ADD and merely warns on DROP. +# `migrate diff --exit-code` compares the LIVE DATABASE against schema.prisma: +# 0 = identical, 2 = they differ, anything else = the CLI itself failed +# (unreachable database, bad DATABASE_URL, schema engine did not start). Those +# last two are different problems and must not share a message — a script whose +# purpose is naming the real cause should not send the on-call hunting for drift +# that was never detected. +# +# ANY difference is fatal, deliberately. An earlier revision tried to classify the +# diff and fail only on missing objects, so that a sibling service mid-rollout +# (which shows up as an extra object) would not block the worker. That classifier +# silently passed real drift — missing enum values and wrong column types both +# escaped it — and printed "schema matches" against a drifted database, which is +# the exact failure mode of the `migrate deploy` bookkeeping this guard exists to +# compensate for. A guard that can be wrong in the reassuring direction is worse +# than no guard, so the crude check stands and OUTPOST_ALLOW_SCHEMA_DRIFT is the +# release valve for the skew case. +# +# Fatal rather than a warning because the worker's sync mappings come from the +# database, and one running against a schema it does not match would write wrong +# statuses to Linear. Failing the deploy keeps the previous replica serving. # -# Deliberately fatal rather than a warning: the worker's sync mappings come from -# the database, and one running against a schema it does not match would write -# wrong statuses to Linear. Failing the deploy keeps the previous replica serving. +# NOTE: this path exits before `exec node`, so nothing binds /health and the +# platform sees only a healthcheck timeout. The reason is in the deploy log, which +# is the only channel available before the process starts. echo "[worker] Checking for schema drift..." set +e -DRIFT_SQL=$($PRISMA migrate diff \ +$PRISMA migrate diff \ --from-schema-datasource "$SCHEMA" \ --to-schema-datamodel "$SCHEMA" \ - --script 2>&1) + --exit-code DRIFT_STATUS=$? set -e -if [ "$DRIFT_STATUS" -ne 0 ]; then - echo "" - echo "$DRIFT_SQL" +if [ "$DRIFT_STATUS" -eq 2 ]; then + case "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" in + 1 | true | TRUE | True | yes | YES | Yes) + echo "" + echo "[worker] WARNING: schema drift detected, but OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting anyway." + echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." + echo "[worker] WARNING: unset this variable as soon as the schema is fixed." + ;; + *) + echo "" + echo "[worker] FATAL: the database does not match schema.prisma (see the diff above)." + echo "[worker] A migration may be recorded as applied without having run." + echo "[worker] Compare models in schema.prisma against the live tables before redeploying." + echo "[worker] If this is transient skew from a sibling service mid-rollout, or you need" + echo "[worker] the worker up during an incident, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." + exit 1 + ;; + esac +elif [ "$DRIFT_STATUS" -ne 0 ]; then echo "" echo "[worker] FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." echo "[worker] This is a tool or connectivity failure, NOT confirmed drift: the database" @@ -64,35 +98,5 @@ if [ "$DRIFT_STATUS" -ne 0 ]; then exit 1 fi -# Statements that CREATE or ADD are objects schema.prisma declares and the database -# lacks — the outage class. Anything else in the diff is an extra object the schema -# does not know about, which the worker does not care about. -MISSING=$(printf '%s\n' "$DRIFT_SQL" | grep -Ei '^[[:space:]]*(CREATE|ALTER[[:space:]]+TABLE.*[[:space:]]ADD[[:space:]])' || true) -EXTRA=$(printf '%s\n' "$DRIFT_SQL" | grep -Ei '^[[:space:]]*DROP' || true) - -if [ -n "$EXTRA" ]; then - echo "[worker] NOTE: the database contains objects schema.prisma does not declare." - echo "[worker] Not fatal — this is normal while a sibling service is mid-rollout." - printf '%s\n' "$EXTRA" | sed 's/^/[worker] /' -fi - -if [ -n "$MISSING" ]; then - echo "" - echo "[worker] Objects schema.prisma declares that the database does not have:" - printf '%s\n' "$MISSING" | sed 's/^/[worker] /' - echo "" - if [ "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" = "1" ]; then - echo "[worker] WARNING: OUTPOST_ALLOW_SCHEMA_DRIFT=1 is set — starting anyway." - echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." - echo "[worker] WARNING: unset this variable as soon as the schema is fixed." - else - echo "[worker] FATAL: the database is missing objects that schema.prisma declares." - echo "[worker] A migration may be recorded as applied without having run." - echo "[worker] Compare models in schema.prisma against the live tables before redeploying." - echo "[worker] To start anyway during an incident, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." - exit 1 - fi -fi - echo "[worker] Migrations complete and schema matches. Starting worker..." exec node apps/worker/dist/index.js diff --git a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql index fe7b8ce1..ce46c75b 100644 --- a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql +++ b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql @@ -1,16 +1,16 @@ -- Repair migration: create SystemConfig where 0001_init did not. -- --- Production has 0001_init recorded as applied in _prisma_migrations, but the --- SystemConfig table it declares does not exist there. Prisma tracks --- migrations by name, so `migrate deploy` reports "no pending migrations" and --- will never create it. Every table and enum in schema.prisma except this one --- is present, so this is the only gap. +-- Production had 0001_init recorded as applied in _prisma_migrations, but the +-- SystemConfig table it declares did not exist there. Prisma tracks migrations +-- by name, so `migrate deploy` reported "no pending migrations" and would never +-- have created it. As observed on 2026-08-12, every other table and enum in +-- schema.prisma was present, so this was the only gap at that time. -- -- The worker reads SystemConfig during boot (buildSyncEngine -> the persisted -- status/priority/label mapping configs) at module scope, before the health --- server starts listening. The read throws, the process exits, nothing ever --- binds /health, and Railway's healthcheck reports "1/1 replicas never became --- healthy". Production has been unable to deploy since 2026-08-07 as a result. +-- server starts listening. The read threw, the process exited, nothing bound +-- /health, and Railway's healthcheck reported "1/1 replicas never became +-- healthy". Production could not deploy from 2026-08-07 to 2026-08-12. -- -- IF NOT EXISTS is deliberate: environments whose 0001_init did create the -- table must no-op rather than fail. Column definitions match the SystemConfig @@ -18,10 +18,10 @@ -- -- It repairs exactly one state — table absent. A SystemConfig that exists with -- the WRONG columns is not repaired: this no-ops, gets recorded as applied, and --- leaves the same "recorded but not effective" gap it was written to close. That --- state is caught at deploy time by the schema-drift guard in --- apps/worker/start.sh, which reports the missing column rather than the missing --- table; repairing it needs an ALTER, not this file. +-- leaves the same "recorded but not effective" gap it was written to close. +-- The schema-drift guard in apps/worker/start.sh fails the deploy on any +-- difference, so that state is detected there rather than here; repairing it +-- needs an ALTER, not this file. CREATE TABLE IF NOT EXISTS "SystemConfig" ( "key" TEXT NOT NULL, "value" TEXT NOT NULL, From 6f1e61bbf06330eb5aade15ec655f71023a92b71 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 10:41:11 -0700 Subject: [PATCH 76/83] fix(worker): address round-3 review; add the CI gate that would have caught this Round 3 (7 agents) on the reduced diff found no behavioural bugs in the shipped logic -- the remaining findings were claim accuracy, input robustness, and one real operational hazard. The hazard: OUTPOST_ALLOW_SCHEMA_DRIFT only covered the drift branch. A CLI-level failure (a schema the pinned CLI rejects, a missing engine binary, an unreadable /opt/prisma) exited 1 with no override at all, which under restartPolicyType="ALWAYS" is an unbreakable crash loop whose only recourse is a code change and a rebuild. A guard added to unblock deploys must not become the thing that blocks them, so the override now covers both fatal branches -- a check that could not RUN is a strictly weaker guarantee than one that ran and found drift, so accepting the latter implies accepting the former. CI now runs the check that was missing. `prisma validate` parses the schema and `migrate deploy` consults only the _prisma_migrations table, so a migration recorded as applied but never effective passed both -- which is why production carried that state for months. The job gets a real Postgres and asserts: - migrations reproduce schema.prisma exactly from empty (migrate diff == 0) - re-applying them is a no-op - with SystemConfig dropped and the repair un-recorded, the guard DETECTS it - the repair migration then fixes it - against a mis-shaped SystemConfig the migration FAILS rather than no-ops - the Dockerfile's Prisma pin matches the lockfile The last two are new behaviour. The migration now asserts its own postcondition with a DO $$ block: CREATE TABLE IF NOT EXISTS silently no-ops on a table with wrong columns and gets recorded as applied, reproducing "recorded but not effective" one level up. Deferring that to the worker's drift guard was not enough either, since apps/web/start.sh runs a bare migrate deploy against the same database with no guard and would close the repair window silently. Failing records P3009 and demands a human -- deliberately louder, because a stopped deploy is recoverable and a silently-ineffective one cost nine days. Also from round 3: - Both prisma invocations are now bounded (timeout, busybox applet; falls back to unbounded with a NOTE if absent). A black-holing endpoint previously hung with no log line at all -- indistinguishable from a slow migration and from a healthy-but-slow boot, defeating the point of putting the reason in the log. - Every FATAL/WARNING goes to stderr. web and worker interleave in Railway logs, so severity has to be machine-distinguishable, not merely prefixed. - Unrecognised override values are echoed back instead of failing closed with output byte-identical to the variable never being set. - The trap comment claimed it aborts a SIGTERM during migrate deploy. Three agents independently reproduced that POSIX sh defers traps behind foreground commands, and the platform signals PID 1 only, so it does not. Narrowed to what it actually buys, with a note that interrupting a migration is deliberately not attempted. - The Dockerfile pin's justification was impossible as written (`prisma@6` cannot resolve 7.9.1; the 7.9.1 sighting was a bare npx). Replaced with the durable reason. apps/web is pinned to 6.19.3 to match, closing the apply-with-one-CLI / verify-with-another loop. - HEALTHCHECK follows the same PORT precedence as index.ts instead of hardcoding 3005, over 127.0.0.1 not localhost, and start-period is 120s because this script now runs two prisma invocations before anything binds /health -- at 10s an orchestrator could restart mid-migration and manufacture the P3009 state. - The migration's scope claim now cites the production-wide migrate diff (which covers indexes and constraints) rather than "every table and enum", and records that staging was not checked. Plus a note that updatedAt has no SQL DEFAULT, so a hand-repair INSERT must pass it explicitly. - OUTPOST_ALLOW_SCHEMA_DRIFT documented in .env.example; start.sh is +x in git. Verified: all nine start.sh branches (clean, drift with/without override in four spellings, tool failure with/without override, deploy failure) exit and start as intended, with FATAL on stderr and unrecognised values echoed; the CI sequence run end-to-end against a throwaway Postgres including the mis-shaped-table failure. turbo typecheck 10/10, turbo test 10/10. --- .env.example | 10 + .github/workflows/ci.yml | 117 ++++++++++- apps/web/Dockerfile | 8 +- apps/worker/Dockerfile | 22 +- apps/worker/start.sh | 190 ++++++++++++------ .../migration.sql | 51 ++++- 6 files changed, 320 insertions(+), 78 deletions(-) diff --git a/.env.example b/.env.example index 0c06ed72..1a0ac502 100644 --- a/.env.example +++ b/.env.example @@ -113,6 +113,16 @@ HEALTH_PORT=3005 # Health check port — read by FIVE services (wor # (discord 3001, slack 3002, teams 3003, worker 3005). # Running several locally needs a per-process override. +# OUTPOST_ALLOW_SCHEMA_DRIFT=1 # Incident release valve for apps/worker/start.sh. + # The worker refuses to boot when the live database does + # not match schema.prisma, or when that check cannot run + # at all. Setting this (1/true/yes/on) starts it anyway, + # with warnings. Use it for transient skew while a sibling + # service is mid-rollout, or to get the worker up during + # an incident; unset it once the schema is repaired. + # TRACKER_SYNC may write wrong statuses to Linear while + # this is set. + # ─── Railway ───────────────────────────────────────────────────────────────── # Railway auto-deploys from GitHub; no deploy hook needed. # RAILWAY_TOKEN is only needed for CLI-based deploys (e.g. `railway up`). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 451be11b..275e5762 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,27 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 + # A real database, because the incident this workflow now guards against was + # invisible without one: `prisma validate` parses the schema and `migrate + # deploy` only consults the _prisma_migrations bookkeeping table, so a + # migration recorded as applied but never executed passes both. Production + # carried exactly that state for months and it only surfaced as nine days of + # "1/1 replicas never became healthy" in August 2026. + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: ci + POSTGRES_PASSWORD: ci + POSTGRES_DB: ci + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U ci" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -48,10 +69,102 @@ jobs: env: DATABASE_URL: 'postgresql://ci:ci@localhost:5432/ci' run: | + set -e pnpm --filter @copilotkit/outpost exec prisma validate --schema db/prisma/schema.prisma - # Ensure migration directory exists with at least one migration test -f packages/outpost/db/prisma/migrations/migration_lock.toml - test -d packages/outpost/db/prisma/migrations/0001_init + + # Apply every migration to a real, empty database, then ask whether the + # resulting schema actually matches schema.prisma. + # + # This is the check that was missing. `migrate deploy` reports success + # from the _prisma_migrations table alone — it never inspects the schema + # — so a migration recorded as applied but never effective is invisible + # to it, and `prisma validate` only parses the file. Production ran in + # exactly that state (0001_init recorded as applied, its SystemConfig + # table absent) until it surfaced as nine days of undiagnosable deploy + # failures in August 2026. Running the same `migrate diff` the worker's + # start.sh uses closes the loop: if the migrations cannot reproduce + # schema.prisma from scratch, this fails here instead of at 3am. + pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + pnpm --filter @copilotkit/outpost exec prisma migrate diff \ + --from-schema-datasource db/prisma/schema.prisma \ + --to-schema-datamodel db/prisma/schema.prisma \ + --exit-code + echo "Migrations reproduce schema.prisma exactly." + + # Applying twice must be a no-op. The repair migration uses + # CREATE TABLE IF NOT EXISTS precisely so environments that already have + # the table do not fail, and this is what proves it. + pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + echo "Re-applying migrations is idempotent." + + - name: Verify the SystemConfig repair path + env: + DATABASE_URL: 'postgresql://ci:ci@localhost:5432/ci' + PGPASSWORD: ci + run: | + set -e + # Reconstruct the broken production state and prove the repair fixes it. + # The step above only covers a fresh database, which was never the broken + # case — production had 0001_init RECORDED AS APPLIED while the table it + # declares was absent, and that is the one scenario this migration exists + # for. Without this, nothing tests it. + psql -h localhost -U ci -d ci -c 'DROP TABLE IF EXISTS "SystemConfig";' + psql -h localhost -U ci -d ci -c \ + "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" + + # The database now looks exactly like production did on 2026-08-07: + # migrations recorded, table missing. The guard must see it... + if pnpm --filter @copilotkit/outpost exec prisma migrate diff \ + --from-schema-datasource db/prisma/schema.prisma \ + --to-schema-datamodel db/prisma/schema.prisma \ + --exit-code; then + echo "FATAL: drift guard did not detect a missing SystemConfig table." + echo "That is the exact blind spot this PR exists to close." + exit 1 + fi + echo "Drift guard detects the missing table." + + # ...and the repair migration must fix it. + pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + pnpm --filter @copilotkit/outpost exec prisma migrate diff \ + --from-schema-datasource db/prisma/schema.prisma \ + --to-schema-datamodel db/prisma/schema.prisma \ + --exit-code + echo "Repair migration restores SystemConfig." + + # A table present but mis-shaped must FAIL the migration loudly rather + # than no-op and be recorded as applied — the "recorded but not + # effective" state that caused the incident. + psql -h localhost -U ci -d ci -c 'DROP TABLE "SystemConfig";' + psql -h localhost -U ci -d ci -c 'CREATE TABLE "SystemConfig" ("key" TEXT PRIMARY KEY);' + psql -h localhost -U ci -d ci -c \ + "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" + if pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma; then + echo "FATAL: migration reported success against a mis-shaped SystemConfig." + exit 1 + fi + echo "Migration fails loudly on a mis-shaped SystemConfig." + + - name: Verify the pinned Prisma CLI matches the lockfile + run: | + set -e + # apps/worker/start.sh gates the worker's boot on this CLI's `migrate + # diff` exit code, so a CLI that has drifted from @prisma/client can + # report false drift and block every deploy with a message pointing at + # the database. Keep the Dockerfile pin and the lockfile in lockstep. + resolved=$(pnpm --filter @copilotkit/outpost exec prisma --version | sed -n 's/^prisma *: *\([0-9][^ ]*\).*/\1/p' | head -1) + pinned=$(sed -n 's/.*--no-save prisma@\([0-9][^ ]*\).*/\1/p' apps/worker/Dockerfile | head -1) + echo "lockfile resolves prisma=$resolved; apps/worker/Dockerfile pins $pinned" + if [ -z "$pinned" ]; then + echo "FATAL: could not read the Prisma pin out of apps/worker/Dockerfile." + exit 1 + fi + if [ "$resolved" != "$pinned" ]; then + echo "FATAL: apps/worker/Dockerfile pins prisma@$pinned but the lockfile resolves $resolved." + echo "Update the Dockerfile pin and the lockfile together." + exit 1 + fi - name: Build run: pnpm build diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 04be8cdd..fe53be8e 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -65,7 +65,13 @@ COPY --from=installer --chown=outpost:outpost /app/node_modules/.pnpm/@prisma+cl # Prisma schema and migrations for migrate deploy on startup COPY --from=installer --chown=outpost:outpost /app/packages/outpost/db/prisma ./packages/outpost/db/prisma # Prisma CLI for running migrations at startup (installed with all transitive deps) -RUN npm install --prefix /opt/prisma --no-save prisma@6 +# Pinned in lockstep with apps/worker/Dockerfile. web and worker run migrate deploy +# against the SAME database, and the worker's start.sh now fails its deploy on any +# schema difference -- so if web applies migrations with a different 6.x CLI than the +# worker verifies with, an unrelated web-side bump can hard-fail worker deploys with a +# drift message describing drift that does not exist. A CI step asserts both literals +# match the lockfile. +RUN npm install --prefix /opt/prisma --no-save prisma@6.19.3 USER outpost ENV NODE_ENV=production diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index f82cfd98..15b52771 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -45,11 +45,12 @@ RUN chmod +x apps/worker/start.sh # Prisma CLI for running migrations at startup (installed standalone, mirrors apps/web/Dockerfile — # the pnpm workspace .bin paths don't resolve reliably in the pruned production image). -# Pinned to the lockfile's version rather than the `prisma@6` range: start.sh now gates -# the worker's boot on this CLI's `migrate diff` exit code, so a CLI change could block -# every deploy. Not hypothetical — resolving the floating range during review picked up -# 7.9.1, which rejects this schema outright. Keep in step with @prisma/client; a -# CLI/client mismatch can itself report false drift. +# Pinned to the lockfile's resolved version rather than the `prisma@6` range, because +# start.sh now gates the worker's boot on this CLI's `migrate diff` exit code: a CLI +# that has drifted from @prisma/client can report false drift and block every deploy +# with a message pointing at the database. `prisma@6` floats freely across 6.x, and +# apps/web applies migrations to this same database with its own CLI — so the two must +# move together. A CI step asserts this literal matches the lockfile. RUN npm install --prefix /opt/prisma --no-save prisma@6.19.3 USER outpost @@ -57,7 +58,14 @@ ENV NODE_ENV=production ENV HEALTH_PORT=3005 EXPOSE 3005 -HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD wget -qO- http://localhost:3005/health || exit 1 +# start-period covers the whole pre-listen window, which is now two full prisma CLI +# invocations (migrate deploy, then the drift check) before anything binds /health. +# At the old 10s an orchestrator honouring this status could restart the container +# mid-migration — manufacturing the failed-migration (P3009) state start.sh exists to +# diagnose. The URL follows the same PORT-over-HEALTH_PORT precedence as src/index.ts +# rather than hardcoding 3005, and uses 127.0.0.1 because `localhost` can resolve to +# ::1 in-container and be refused (see apps/web/Dockerfile). +HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \ + CMD wget -qO- "http://127.0.0.1:${PORT:-${HEALTH_PORT:-3003}}/health" || exit 1 CMD ["./apps/worker/start.sh"] diff --git a/apps/worker/start.sh b/apps/worker/start.sh index edfac446..60adcd91 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -1,27 +1,83 @@ #!/bin/sh -set -e +set -eu -# Until `exec node` below, PID 1 is this shell. Linux discards signals that PID 1 -# has no handler for, so without this a SIGTERM arriving during migrate deploy is -# ignored outright and the platform waits out the full grace period before -# SIGKILLing — potentially mid-migration. -trap 'echo "[worker] received SIGTERM during startup, aborting"; exit 143' TERM -trap 'echo "[worker] received SIGINT during startup, aborting"; exit 130' INT +# These fire only BETWEEN the startup steps below, not during them: POSIX sh defers +# a trap until the running foreground command returns, and the platform signals PID +# 1 only, so `prisma` never receives it either. A SIGTERM arriving mid-migration is +# therefore NOT aborted — the shell waits for prisma to finish, then runs this. +# What that buys is worth having anyway: the script will not fall through to +# `exec node` after an aborted shutdown, and the abort is logged rather than silent. +# Interrupting a migration in flight is deliberately not attempted; a half-applied +# migration is the P3009 state this script exists to diagnose. +trap 'echo "[worker] SIGTERM received between startup steps, aborting" >&2; exit 143' TERM +trap 'echo "[worker] SIGINT received between startup steps, aborting" >&2; exit 130' INT PRISMA="node /opt/prisma/node_modules/prisma/build/index.js" SCHEMA="packages/outpost/db/prisma/schema.prisma" +# Neither prisma invocation is bounded by default. Against a black-holing endpoint +# (a dropped packet rather than a refusal) the CLI can hang far past any TCP +# timeout, and a hang produces NO log line at all — indistinguishable from a slow +# migration and from a healthy-but-slow boot, which defeats the point of putting +# the reason in the deploy log. `timeout` is a busybox applet, present in alpine. +STEP_TIMEOUT="${OUTPOST_STARTUP_STEP_TIMEOUT:-180}" + +if command -v timeout >/dev/null 2>&1; then + run_step() { timeout "$STEP_TIMEOUT" "$@"; } +else + # Not the deployed path (busybox provides timeout in node:*-alpine), but a + # missing applet must not fail every startup step closed. + echo "[worker] NOTE: timeout(1) is unavailable; startup steps will run unbounded." >&2 + run_step() { "$@"; } +fi + +# Everything diagnostic goes to stderr. web and worker migrate into the same +# database and their Railway logs interleave, so severity has to be +# machine-distinguishable and not merely prefixed — log shippers and alert rules +# key off the stream, not the text. +fatal() { + echo "" >&2 + while [ "$#" -gt 0 ]; do + echo "[worker] $1" >&2 + shift + done +} + +# Recognised spellings of the incident override. Anything else fails closed, but +# says so — an operator who sets OUTPOST_ALLOW_SCHEMA_DRIFT=on under pressure must +# not get output byte-identical to having never set it at all. +drift_override_set() { + value=$(printf '%s' "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]"') + case "$value" in + 1 | true | yes | y | on | enabled) return 0 ;; + '' | 0 | false | no | n | off) return 1 ;; + *) + echo "[worker] NOTE: OUTPOST_ALLOW_SCHEMA_DRIFT is set to '${OUTPOST_ALLOW_SCHEMA_DRIFT:-}', which is not recognised." >&2 + echo "[worker] NOTE: use 1, true, yes, or on. Treating it as NOT set." >&2 + return 1 + ;; + esac +} + echo "[worker] Running database migrations..." # A migrate deploy failure is far more common than drift and, left to bare `set -e`, # dies with Prisma's output and no framing — in a script whose whole purpose is -# naming the real cause. web and worker migrate into the same database and their -# logs interleave, hence the [worker] prefixes throughout. -if ! $PRISMA migrate deploy --schema "$SCHEMA"; then - echo "" - echo "[worker] FATAL: prisma migrate deploy failed (see the error above)." - echo "[worker] Common causes: a migration recorded as failed in _prisma_migrations (P3009)," - echo "[worker] a migration file edited after it was applied (P3006), or an unreachable database." - echo "[worker] The worker will not start against a database whose migrations did not apply." +# naming the real cause. +set +e +run_step $PRISMA migrate deploy --schema "$SCHEMA" +DEPLOY_STATUS=$? +set -e + +if [ "$DEPLOY_STATUS" -eq 124 ]; then + fatal "FATAL: prisma migrate deploy timed out after ${STEP_TIMEOUT}s." \ + "The database accepted the connection but did not finish the migration." \ + "Check for a lock held by another migrating service, or a database under load." + exit 1 +elif [ "$DEPLOY_STATUS" -ne 0 ]; then + fatal "FATAL: prisma migrate deploy failed (see the error above)." \ + "Common causes: a migration recorded as failed in _prisma_migrations (P3009)," \ + "a migration file edited after it was applied (P3006), or an unreachable database." \ + "The worker will not start against a database whose migrations did not apply." exit 1 fi @@ -37,34 +93,34 @@ fi # while the SystemConfig table it declares did not exist. Every deploy from # 2026-08-07 failed with nothing but "1/1 replicas never became healthy" — nine # days of a five-minute healthcheck timeout that looked like a broken image. +# CI now runs this same comparison against a real database on every PR, so the +# repo-side version of that gap fails there rather than here. # -# `migrate diff --exit-code` compares the LIVE DATABASE against schema.prisma: -# 0 = identical, 2 = they differ, anything else = the CLI itself failed -# (unreachable database, bad DATABASE_URL, schema engine did not start). Those -# last two are different problems and must not share a message — a script whose -# purpose is naming the real cause should not send the on-call hunting for drift -# that was never detected. +# `migrate diff --exit-code`: 0 = identical, 2 = they differ, anything else = the +# CLI itself failed. Those last two are different problems and must not share a +# message — a script whose purpose is naming the real cause should not send the +# on-call hunting for drift that was never detected. # -# ANY difference is fatal, deliberately. An earlier revision tried to classify the -# diff and fail only on missing objects, so that a sibling service mid-rollout -# (which shows up as an extra object) would not block the worker. That classifier -# silently passed real drift — missing enum values and wrong column types both -# escaped it — and printed "schema matches" against a drifted database, which is -# the exact failure mode of the `migrate deploy` bookkeeping this guard exists to -# compensate for. A guard that can be wrong in the reassuring direction is worse -# than no guard, so the crude check stands and OUTPOST_ALLOW_SCHEMA_DRIFT is the -# release valve for the skew case. +# ANY difference is fatal, deliberately. An earlier revision classified the diff +# and failed only on missing objects, so a sibling service mid-rollout would not +# block the worker. That classifier silently passed real drift — missing enum +# values and wrong column types both escaped it — and printed "schema matches" +# against a drifted database, which is the exact failure mode of the `migrate +# deploy` bookkeeping this guard compensates for. A guard that can be wrong in the +# reassuring direction is worse than no guard, so the crude check stands and +# OUTPOST_ALLOW_SCHEMA_DRIFT is the release valve for the skew case. # -# Fatal rather than a warning because the worker's sync mappings come from the -# database, and one running against a schema it does not match would write wrong -# statuses to Linear. Failing the deploy keeps the previous replica serving. +# Fatal because the worker's sync mappings come from the database, and one running +# against a schema it does not match would write wrong statuses to Linear. Failing +# the deploy leaves the previously-deployed replica running — note that replica is +# already serving against this same database and was never re-checked, so this +# buys "no NEW bad worker", not "the database is fine". # -# NOTE: this path exits before `exec node`, so nothing binds /health and the -# platform sees only a healthcheck timeout. The reason is in the deploy log, which -# is the only channel available before the process starts. +# NOTE: every exit below happens before `exec node`, so nothing binds /health and +# the platform sees only a healthcheck timeout. The deploy log is the sole channel. echo "[worker] Checking for schema drift..." set +e -$PRISMA migrate diff \ +run_step $PRISMA migrate diff \ --from-schema-datasource "$SCHEMA" \ --to-schema-datamodel "$SCHEMA" \ --exit-code @@ -72,31 +128,43 @@ DRIFT_STATUS=$? set -e if [ "$DRIFT_STATUS" -eq 2 ]; then - case "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" in - 1 | true | TRUE | True | yes | YES | Yes) - echo "" - echo "[worker] WARNING: schema drift detected, but OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting anyway." - echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." - echo "[worker] WARNING: unset this variable as soon as the schema is fixed." - ;; - *) - echo "" - echo "[worker] FATAL: the database does not match schema.prisma (see the diff above)." - echo "[worker] A migration may be recorded as applied without having run." - echo "[worker] Compare models in schema.prisma against the live tables before redeploying." - echo "[worker] If this is transient skew from a sibling service mid-rollout, or you need" - echo "[worker] the worker up during an incident, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." - exit 1 - ;; - esac + if drift_override_set; then + echo "[worker] WARNING: schema drift detected, but OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting anyway." >&2 + echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." >&2 + echo "[worker] WARNING: unset this variable as soon as the schema is fixed." >&2 + else + fatal "FATAL: the database does not match schema.prisma (see the diff above)." \ + "A migration may be recorded as applied without having run." \ + "Compare models in schema.prisma against the live tables before redeploying." \ + "If this is transient skew from a sibling service mid-rollout, or you need the" \ + "worker up during an incident, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." + exit 1 + fi elif [ "$DRIFT_STATUS" -ne 0 ]; then - echo "" - echo "[worker] FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." - echo "[worker] This is a tool or connectivity failure, NOT confirmed drift: the database" - echo "[worker] was never successfully compared. Check DATABASE_URL and that the database" - echo "[worker] is reachable from this container, then redeploy." - exit 1 + # The override covers this branch too. A check that could not RUN is a strictly + # weaker guarantee than one that ran and found drift, so an operator who accepts + # drift necessarily accepts this. Without it, any CLI-level breakage — a schema + # the pinned CLI rejects, a missing engine binary, an unreadable /opt/prisma — + # is an unbreakable crash loop under restartPolicyType="ALWAYS", with no recourse + # short of a code change and a rebuild. A guard added to unblock deploys must not + # become the thing that blocks them. + if [ "$DRIFT_STATUS" -eq 124 ]; then + fatal "FATAL: the schema-drift check timed out after ${STEP_TIMEOUT}s." \ + "The database accepted the connection but never answered. This is NOT confirmed drift." + else + fatal "FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." \ + "The database was NEVER COMPARED. This is NOT confirmed drift." \ + "Causes, roughly by likelihood: DATABASE_URL unset or wrong; database unreachable" \ + "from this container; the pinned prisma CLI (apps/worker/Dockerfile) rejecting" \ + "schema.prisma; a missing or unloadable schema engine binary." + fi + if drift_override_set; then + echo "[worker] WARNING: OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting unverified." >&2 + else + echo "[worker] To boot without the guard, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." >&2 + exit 1 + fi fi -echo "[worker] Migrations complete and schema matches. Starting worker..." +echo "[worker] Migrations complete and schema verified. Starting worker..." exec node apps/worker/dist/index.js diff --git a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql index ce46c75b..6d9d4615 100644 --- a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql +++ b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql @@ -3,8 +3,16 @@ -- Production had 0001_init recorded as applied in _prisma_migrations, but the -- SystemConfig table it declares did not exist there. Prisma tracks migrations -- by name, so `migrate deploy` reported "no pending migrations" and would never --- have created it. As observed on 2026-08-12, every other table and enum in --- schema.prisma was present, so this was the only gap at that time. +-- have created it. +-- +-- Scope of the gap, as established on 2026-08-12: after the table was created +-- directly in production, a full `migrate diff --from-schema-datasource +-- --to-schema-datamodel` against that database reported "No difference detected". +-- That compares indexes, constraints and column types, not just tables — so +-- SystemConfig really was the only difference, and the rest of 0001_init (which +-- creates 45 indexes and 12 foreign keys AFTER this table) did land. Staging was +-- not checked; its Postgres has no public URL, and the drift guard in +-- apps/worker/start.sh is what will report the answer on its next deploy. -- -- The worker reads SystemConfig during boot (buildSyncEngine -> the persisted -- status/priority/label mapping configs) at module scope, before the health @@ -17,14 +25,43 @@ -- block in 0001_init exactly. -- -- It repairs exactly one state — table absent. A SystemConfig that exists with --- the WRONG columns is not repaired: this no-ops, gets recorded as applied, and --- leaves the same "recorded but not effective" gap it was written to close. --- The schema-drift guard in apps/worker/start.sh fails the deploy on any --- difference, so that state is detected there rather than here; repairing it --- needs an ALTER, not this file. +-- the WRONG columns cannot be repaired by a CREATE; that needs an ALTER, and +-- guessing the right one blind is worse than stopping. +-- Note for anyone hand-repairing this table under pressure: updatedAt is NOT NULL +-- with no DEFAULT, because @updatedAt is applied client-side by Prisma. A bare +-- `INSERT INTO "SystemConfig" (key, value) VALUES (...)` from psql will fail the +-- not-null check; pass updatedAt = now() explicitly. CREATE TABLE IF NOT EXISTS "SystemConfig" ( "key" TEXT NOT NULL, "value" TEXT NOT NULL, "updatedAt" TIMESTAMP(3) NOT NULL, CONSTRAINT "SystemConfig_pkey" PRIMARY KEY ("key") ); + +-- Assert the repair actually took, rather than trusting it. +-- +-- Left alone, the wrong-columns case is the failure this migration was written to +-- eliminate, reproduced one level up: CREATE ... IF NOT EXISTS no-ops, Prisma +-- records the migration as applied, and the database still does not match +-- schema.prisma — "recorded but not effective" all over again. Deferring it to the +-- worker's drift guard is not enough either, because apps/web/start.sh runs a bare +-- `migrate deploy` against this same database with no guard, so a web-first deploy +-- would close the repair window silently and leave detection to whenever the +-- worker next deploys. +-- +-- Failing here instead records the migration as FAILED (P3009), which blocks both +-- services and demands a human. That is deliberately louder than the alternative: +-- a stopped deploy is recoverable, a silently-ineffective one cost nine days. +DO $$ +BEGIN + IF ( + SELECT count(*) + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'SystemConfig' + AND column_name IN ('key', 'value', 'updatedAt') + ) <> 3 THEN + RAISE EXCEPTION + 'SystemConfig exists with unexpected columns; repairing it needs an ALTER, not this migration. Compare the live table against the SystemConfig model in schema.prisma.'; + END IF; +END $$; From 4c3a5f6720fe8624f2e91dbd37b95c44b06595c3 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 10:57:43 -0700 Subject: [PATCH 77/83] fix(worker): address round-4 review; test the guard itself in CI Round 4 (7 agents) found no behavioural bugs in the migration or the guard's decision logic, but three defects in the machinery around them -- two of which would have made the guard lie in the reassuring direction, which is the failure mode this whole change argues is worse than no guard. BUSYBOX TIMEOUT SEMANTICS Both timeout branches were dead code. The runtime is node:20-alpine, whose busybox timeout reports 128+SIGTERM = 143, not GNU coreutils' 124 -- verified in the actual image by four agents independently. So a black-holed database (the exact case the timeout was added for) fell through to the generic branch and was reported as "migrate deploy failed ... P3009 / P3006 / unreachable database", none of which was the cause. Without -k, busybox also signals and then waits for the child anyway, so a step that traps SIGTERM (the prisma CLI does) had no bound at all. run_step now uses -k and normalises 143/137 to 124. migrate deploy is no longer bounded. Killing it mid-migration leaves _prisma_migrations holding finished_at = NULL -- the P3009 state this script exists to diagnose -- and an agent reproduced the full sequence: killed deploy, orphaned pg_advisory_lock, then P1002 on the next attempt and P3009 after that, blocking BOTH services. The bound only makes sense on the read-only drift check, and the signal traps already document that interrupting a migration is deliberately not attempted. The two policies now agree. THE MIGRATION'S ASSERTION WAS MUCH WEAKER THAN ITS COMMENT It counted column NAMES. A SystemConfig with all three names but wrong types, or nullable, or carrying an extra column, or missing its primary key (which Prisma's upsert on @id key requires) passed, no-opped, and was recorded as applied -- the "recorded but not effective" state one level up, again. It now checks types, nullability, exact column count and the PK. It also queried information_schema with a hardcoded 'public'. Both were wrong: information_schema is privilege-filtered, so a table created by another role -- which is exactly how production's was created -- could read as absent and hard-fail into P3009 over a database that was fine; and the CREATE is unqualified so it follows search_path, meaning any ?schema= environment would inspect the wrong schema. Now pg_catalog and current_schema(). CI NOW RUNS THE GUARD ITSELF Nothing executed start.sh. CI re-implemented its prisma calls in bash on ubuntu, which is precisely why an alpine-only exit-code difference sat in it unnoticed. PRISMA is overridable via OUTPOST_PRISMA_CMD so the script runs against a stub, and eleven cases now assert exit status and whether the worker boots: clean, drift blocked, drift overridden in two spellings, unknown and falsey overrides blocked, tool failure blocked and overridden, deploy failure, plus a non-numeric and a zero timeout. Two more assert diagnostics do not leak to stdout and that an overridden boot does not emit FATAL. Also fixed, all from round 4: - The pin check read only apps/worker/Dockerfile while apps/web/Dockerfile's comment claimed both were covered. It loops over both now, anchored on ^RUN, with pipefail and a guard on the parsed version. - Both CI negative controls accepted any non-zero exit, so a broken CLI or an unreachable database would have passed as "the guard works". They assert exit 2 specifically, and the mis-shaped case greps for the migration's own message. - The idempotency step was vacuous: migrate deploy skips recorded migrations, so it ran zero SQL and would have passed with a body of SELECT 1/0. It now deletes the row first, which also covers the majority path (table present, repair unrecorded) that nothing tested. - CI restores the database after the destructive assertions, which previously left a mis-shaped table and a failed P3009 row for every later step. - FATAL was emitted before the override was consulted, so an overridden boot paged. And the success line claimed "schema verified" on both override paths, including the one where nothing was compared. - OUTPOST_STARTUP_STEP_TIMEOUT is validated; 0 now means unbounded (the GNU reading an operator intends) rather than busybox's kill-immediately. Default lowered to 120s and documented in .env.example. - Corrected the index count in the migration's audit comment (27, not 45). Verified against a throwaway Postgres: wrong types, nullable columns, an extra column, and a missing primary key each fail the migration; absent and correct tables both succeed; a fresh deploy into a non-public schema succeeds. The CI guard-test block was run verbatim -- all eleven cases pass. turbo typecheck 10/10, turbo test 10/10. --- .env.example | 9 ++ .github/workflows/ci.yml | 137 +++++++++++++++--- apps/worker/start.sh | 111 +++++++++----- .../migration.sql | 44 ++++-- 4 files changed, 240 insertions(+), 61 deletions(-) diff --git a/.env.example b/.env.example index 1a0ac502..2d476a34 100644 --- a/.env.example +++ b/.env.example @@ -123,6 +123,15 @@ HEALTH_PORT=3005 # Health check port — read by FIVE services (wor # TRACKER_SYNC may write wrong statuses to Linear while # this is set. +# OUTPOST_STARTUP_STEP_TIMEOUT=180 # Seconds the worker's schema-drift check may run + # before it is abandoned (apps/worker/start.sh). Bounds + # only the read-only check; `migrate deploy` is left + # unbounded on purpose, since killing it mid-migration + # creates the P3009 failed-migration state the guard + # exists to report. Keep it below the platform's + # healthcheck timeout, or a slow check is killed as an + # unexplained deploy failure instead of logging a reason. + # ─── Railway ───────────────────────────────────────────────────────────────── # Railway auto-deploys from GitHub; no deploy hook needed. # RAILWAY_TOKEN is only needed for CLI-based deploys (e.g. `railway up`). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275e5762..6bcc0c81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,11 +92,16 @@ jobs: --exit-code echo "Migrations reproduce schema.prisma exactly." - # Applying twice must be a no-op. The repair migration uses - # CREATE TABLE IF NOT EXISTS precisely so environments that already have - # the table do not fail, and this is what proves it. + # Re-running `migrate deploy` proves nothing on its own: the migration + # rows already exist, so it applies zero SQL and would pass even if the + # migration body were `SELECT 1/0`. That is the same bookkeeping-only + # blind spot this whole job exists to close. To actually exercise + # CREATE TABLE IF NOT EXISTS, the row has to be removed while the + # correctly-shaped table stays. + psql -h localhost -U ci -d ci -q -c \ + "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma - echo "Re-applying migrations is idempotent." + echo "Repair migration re-runs cleanly against an already-correct table." - name: Verify the SystemConfig repair path env: @@ -115,15 +120,27 @@ jobs: # The database now looks exactly like production did on 2026-08-07: # migrations recorded, table missing. The guard must see it... - if pnpm --filter @copilotkit/outpost exec prisma migrate diff \ + # Assert exit 2 SPECIFICALLY. `migrate diff --exit-code` returns 0 for + # identical, 2 for a difference, and 1 (or other) when the CLI itself + # failed — so `if ! ...` would accept a broken CLI or a lost connection + # as proof the guard works, having never compared anything. start.sh + # treats that distinction as load-bearing; the test for it must too. + set +e + pnpm --filter @copilotkit/outpost exec prisma migrate diff \ --from-schema-datasource db/prisma/schema.prisma \ --to-schema-datamodel db/prisma/schema.prisma \ - --exit-code; then + --exit-code + drift_status=$? + set -e + if [ "$drift_status" -eq 0 ]; then echo "FATAL: drift guard did not detect a missing SystemConfig table." echo "That is the exact blind spot this PR exists to close." exit 1 + elif [ "$drift_status" -ne 2 ]; then + echo "FATAL: migrate diff exited $drift_status — it never compared the database." + exit 1 fi - echo "Drift guard detects the missing table." + echo "Drift guard detects the missing table (exit 2)." # ...and the repair migration must fix it. pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma @@ -137,15 +154,86 @@ jobs: # than no-op and be recorded as applied — the "recorded but not # effective" state that caused the incident. psql -h localhost -U ci -d ci -c 'DROP TABLE "SystemConfig";' - psql -h localhost -U ci -d ci -c 'CREATE TABLE "SystemConfig" ("key" TEXT PRIMARY KEY);' + # Wrong TYPE, not just a missing column name — the assertion must check + # shape, since a table with the right three names but wrong types is + # still a database that does not match schema.prisma. + psql -h localhost -U ci -d ci -c 'CREATE TABLE "SystemConfig" ("key" TEXT PRIMARY KEY, "value" INTEGER NOT NULL, "updatedAt" TIMESTAMP(3) NOT NULL);' psql -h localhost -U ci -d ci -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" - if pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma; then + set +e + deploy_out=$(pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma 2>&1) + deploy_status=$? + set -e + echo "$deploy_out" + if [ "$deploy_status" -eq 0 ]; then echo "FATAL: migration reported success against a mis-shaped SystemConfig." exit 1 fi + # Any non-zero would also cover "database unreachable", which would pass + # this test without the assertion ever firing. Require our own message. + case "$deploy_out" in + *"SystemConfig exists with unexpected columns"*) ;; + *) + echo "FATAL: deploy failed, but not via the migration's own guard." + exit 1 + ;; + esac echo "Migration fails loudly on a mis-shaped SystemConfig." + # Leave the database usable. These assertions deliberately corrupt it + # (mis-shaped table plus a failed P3009 row), and they run before Build + # and Test — so the first DB-touching test added to this job would + # otherwise inherit a knowingly-broken schema. + psql -h localhost -U ci -d ci -q -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' + pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + echo "CI database restored." + + - name: Verify the worker startup guard + run: | + set -e + # Execute start.sh itself against a stub CLI. Nothing previously ran + # this script -- CI re-implemented its prisma calls in bash -- which is + # why a busybox-vs-GNU timeout exit-code difference sat in it unnoticed. + # Every branch that decides whether the worker may boot is asserted here. + stub=$(mktemp); chmod +x "$stub" + printf '#!/bin/sh\nif [ "$1" = migrate ] && [ "$2" = diff ]; then exit "${DIFF_RC:-0}"; fi\nexit "${DEPLOY_RC:-0}"\n' > "$stub" + script=$(mktemp) + sed 's|^exec node apps/worker/dist/index.js|echo STARTED|' apps/worker/start.sh > "$script" + + check() { # label expect_exit expect_started env... + label=$1 want_rc=$2 want_started=$3; shift 3 + out=$(env OUTPOST_PRISMA_CMD="$stub" "$@" sh "$script" 2>&1); rc=$? + case "$out" in *STARTED*) got=yes ;; *) got=no ;; esac + if [ "$rc" != "$want_rc" ] || [ "$got" != "$want_started" ]; then + echo "FATAL: $label -> exit=$rc started=$got (wanted exit=$want_rc started=$want_started)" + echo "$out" + exit 1 + fi + echo " ok: $label" + } + + check "clean schema boots" 0 yes DIFF_RC=0 + check "drift blocks the boot" 1 no DIFF_RC=2 + check "drift + override boots" 0 yes DIFF_RC=2 OUTPOST_ALLOW_SCHEMA_DRIFT=1 + check "drift + override=on boots" 0 yes DIFF_RC=2 OUTPOST_ALLOW_SCHEMA_DRIFT=on + check "drift + unknown override blocks" 1 no DIFF_RC=2 OUTPOST_ALLOW_SCHEMA_DRIFT=banana + check "drift + override=0 blocks" 1 no DIFF_RC=2 OUTPOST_ALLOW_SCHEMA_DRIFT=0 + # A check that could not RUN must be overridable too, or a CLI-level + # breakage is an unbreakable crash loop under restartPolicyType=ALWAYS. + check "tool failure blocks" 1 no DIFF_RC=1 + check "tool failure + override boots" 0 yes DIFF_RC=1 OUTPOST_ALLOW_SCHEMA_DRIFT=1 + check "migrate deploy failure blocks" 1 no DEPLOY_RC=1 + # A bad timeout value must not fail the step closed. + check "non-numeric timeout still boots" 0 yes DIFF_RC=0 OUTPOST_STARTUP_STEP_TIMEOUT=abc + check "timeout=0 means unbounded" 0 yes DIFF_RC=0 OUTPOST_STARTUP_STEP_TIMEOUT=0 + + # Diagnostics must reach stderr, and an overridden boot must not page. + env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=2 sh "$script" 2>/dev/null | grep -q FATAL \ + && { echo "FATAL: diagnostics leaked to stdout."; exit 1; } + env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=1 OUTPOST_ALLOW_SCHEMA_DRIFT=1 sh "$script" 2>&1 >/dev/null | grep -q FATAL \ + && { echo "FATAL: an overridden boot emitted FATAL and would page."; exit 1; } + echo "Startup guard behaves correctly across all branches." + - name: Verify the pinned Prisma CLI matches the lockfile run: | set -e @@ -153,18 +241,31 @@ jobs: # diff` exit code, so a CLI that has drifted from @prisma/client can # report false drift and block every deploy with a message pointing at # the database. Keep the Dockerfile pin and the lockfile in lockstep. + set -o pipefail resolved=$(pnpm --filter @copilotkit/outpost exec prisma --version | sed -n 's/^prisma *: *\([0-9][^ ]*\).*/\1/p' | head -1) - pinned=$(sed -n 's/.*--no-save prisma@\([0-9][^ ]*\).*/\1/p' apps/worker/Dockerfile | head -1) - echo "lockfile resolves prisma=$resolved; apps/worker/Dockerfile pins $pinned" - if [ -z "$pinned" ]; then - echo "FATAL: could not read the Prisma pin out of apps/worker/Dockerfile." - exit 1 - fi - if [ "$resolved" != "$pinned" ]; then - echo "FATAL: apps/worker/Dockerfile pins prisma@$pinned but the lockfile resolves $resolved." - echo "Update the Dockerfile pin and the lockfile together." + if [ -z "$resolved" ]; then + echo "FATAL: could not parse a version from 'prisma --version'; its output format may have changed." exit 1 fi + echo "lockfile resolves prisma=$resolved" + + # BOTH images, not just the worker's. web and worker apply/verify + # migrations against the same database, so a bump to either alone + # recreates the CLI skew this check exists to prevent. Anchored on ^RUN + # so a comment quoting the command cannot shadow the real line. + for dockerfile in apps/worker/Dockerfile apps/web/Dockerfile; do + pinned=$(sed -n 's/^RUN .*--no-save prisma@\([0-9][^ ]*\).*/\1/p' "$dockerfile" | head -1) + if [ -z "$pinned" ]; then + echo "FATAL: could not read a Prisma pin out of $dockerfile." + exit 1 + fi + if [ "$resolved" != "$pinned" ]; then + echo "FATAL: $dockerfile pins prisma@$pinned but the lockfile resolves $resolved." + echo "Update the Dockerfile pins and the lockfile together." + exit 1 + fi + echo " $dockerfile pins $pinned — matches." + done - name: Build run: pnpm build diff --git a/apps/worker/start.sh b/apps/worker/start.sh index 60adcd91..fc7c9e32 100644 --- a/apps/worker/start.sh +++ b/apps/worker/start.sh @@ -12,29 +12,64 @@ set -eu trap 'echo "[worker] SIGTERM received between startup steps, aborting" >&2; exit 143' TERM trap 'echo "[worker] SIGINT received between startup steps, aborting" >&2; exit 130' INT -PRISMA="node /opt/prisma/node_modules/prisma/build/index.js" +# Overridable so CI can execute this script against a stub that returns canned exit +# codes. Nothing previously ran start.sh at all — CI re-implemented the prisma calls +# in bash on ubuntu, which is exactly why a busybox-vs-GNU exit-code difference in +# the timeout handling went unnoticed. +PRISMA="${OUTPOST_PRISMA_CMD:-node /opt/prisma/node_modules/prisma/build/index.js}" SCHEMA="packages/outpost/db/prisma/schema.prisma" -# Neither prisma invocation is bounded by default. Against a black-holing endpoint -# (a dropped packet rather than a refusal) the CLI can hang far past any TCP -# timeout, and a hang produces NO log line at all — indistinguishable from a slow -# migration and from a healthy-but-slow boot, which defeats the point of putting -# the reason in the deploy log. `timeout` is a busybox applet, present in alpine. -STEP_TIMEOUT="${OUTPOST_STARTUP_STEP_TIMEOUT:-180}" +# The drift check is bounded. Against a black-holing endpoint (a dropped packet +# rather than a refusal) the CLI can hang far past any TCP timeout, and a hang +# produces NO log line at all — indistinguishable from a slow migration and from a +# healthy-but-slow boot, which defeats the point of putting the reason in the log. +# +# `migrate deploy` is deliberately NOT bounded. Killing it mid-migration leaves +# _prisma_migrations holding a row with finished_at = NULL, which is the P3009 +# failed-migration state this script exists to diagnose — a bound there would +# manufacture the very condition it reports. Same reasoning as the signal traps +# above: a slow migration is survivable, a half-applied one is not. +STEP_TIMEOUT="${OUTPOST_STARTUP_STEP_TIMEOUT:-120}" +UNBOUNDED=no +case "$STEP_TIMEOUT" in + '' | *[!0-9]*) + echo "[worker] NOTE: OUTPOST_STARTUP_STEP_TIMEOUT='${STEP_TIMEOUT}' is not a whole number of seconds; using 120." >&2 + STEP_TIMEOUT=120 + ;; + 0) + # GNU timeout reads 0 as "no limit"; busybox reads it as "kill immediately", + # which would SIGKILL the drift check at t=0 and report it as a database + # failure. Honour the GNU reading, since that is what an operator means. + echo "[worker] NOTE: OUTPOST_STARTUP_STEP_TIMEOUT=0 — the drift check will run unbounded." >&2 + UNBOUNDED=yes + ;; +esac -if command -v timeout >/dev/null 2>&1; then - run_step() { timeout "$STEP_TIMEOUT" "$@"; } +if [ "$UNBOUNDED" = no ] && command -v timeout >/dev/null 2>&1; then + # -k so a child that traps SIGTERM (the prisma CLI does, for engine cleanup) is + # still killed: busybox otherwise signals and then waits for it anyway, which + # means no bound at all. And busybox reports a timeout as 128+SIGTERM = 143, + # not GNU coreutils' 124 — normalise here so callers can test one value. + run_step() { + timeout -k 10 "$STEP_TIMEOUT" "$@" + _st=$? + if [ "$_st" -eq 143 ] || [ "$_st" -eq 137 ]; then + return 124 + fi + return "$_st" + } else # Not the deployed path (busybox provides timeout in node:*-alpine), but a # missing applet must not fail every startup step closed. - echo "[worker] NOTE: timeout(1) is unavailable; startup steps will run unbounded." >&2 + [ "$UNBOUNDED" = no ] && echo "[worker] NOTE: timeout(1) is unavailable; the drift check will run unbounded." >&2 run_step() { "$@"; } fi -# Everything diagnostic goes to stderr. web and worker migrate into the same -# database and their Railway logs interleave, so severity has to be -# machine-distinguishable and not merely prefixed — log shippers and alert rules -# key off the stream, not the text. +# Progress goes to stdout; everything else (NOTE, WARNING, FATAL) to stderr. web +# and worker migrate into the same database and their Railway logs interleave, so +# "something needs attention" has to be machine-distinguishable and not merely +# prefixed. Note this is a two-way split, not three severity levels — an alert rule +# keyed on stderr alone will also catch NOTE lines. fatal() { echo "" >&2 while [ "$#" -gt 0 ]; do @@ -47,8 +82,8 @@ fatal() { # says so — an operator who sets OUTPOST_ALLOW_SCHEMA_DRIFT=on under pressure must # not get output byte-identical to having never set it at all. drift_override_set() { - value=$(printf '%s' "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]"') - case "$value" in + _drift_value=$(printf '%s' "${OUTPOST_ALLOW_SCHEMA_DRIFT:-0}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]"') + case "$_drift_value" in 1 | true | yes | y | on | enabled) return 0 ;; '' | 0 | false | no | n | off) return 1 ;; *) @@ -64,16 +99,11 @@ echo "[worker] Running database migrations..." # dies with Prisma's output and no framing — in a script whose whole purpose is # naming the real cause. set +e -run_step $PRISMA migrate deploy --schema "$SCHEMA" +$PRISMA migrate deploy --schema "$SCHEMA" DEPLOY_STATUS=$? set -e -if [ "$DEPLOY_STATUS" -eq 124 ]; then - fatal "FATAL: prisma migrate deploy timed out after ${STEP_TIMEOUT}s." \ - "The database accepted the connection but did not finish the migration." \ - "Check for a lock held by another migrating service, or a database under load." - exit 1 -elif [ "$DEPLOY_STATUS" -ne 0 ]; then +if [ "$DEPLOY_STATUS" -ne 0 ]; then fatal "FATAL: prisma migrate deploy failed (see the error above)." \ "Common causes: a migration recorded as failed in _prisma_migrations (P3009)," \ "a migration file edited after it was applied (P3006), or an unreachable database." \ @@ -118,6 +148,7 @@ fi # # NOTE: every exit below happens before `exec node`, so nothing binds /health and # the platform sees only a healthcheck timeout. The deploy log is the sole channel. +SCHEMA_VERIFIED=yes echo "[worker] Checking for schema drift..." set +e run_step $PRISMA migrate diff \ @@ -132,8 +163,9 @@ if [ "$DRIFT_STATUS" -eq 2 ]; then echo "[worker] WARNING: schema drift detected, but OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting anyway." >&2 echo "[worker] WARNING: TRACKER_SYNC may write wrong statuses to Linear until this is repaired." >&2 echo "[worker] WARNING: unset this variable as soon as the schema is fixed." >&2 + SCHEMA_VERIFIED=no else - fatal "FATAL: the database does not match schema.prisma (see the diff above)." \ + fatal "FATAL: the database does not match schema.prisma (migrate diff printed the differences on stdout, above)." \ "A migration may be recorded as applied without having run." \ "Compare models in schema.prisma against the live tables before redeploying." \ "If this is transient skew from a sibling service mid-rollout, or you need the" \ @@ -149,22 +181,33 @@ elif [ "$DRIFT_STATUS" -ne 0 ]; then # short of a code change and a rebuild. A guard added to unblock deploys must not # become the thing that blocks them. if [ "$DRIFT_STATUS" -eq 124 ]; then - fatal "FATAL: the schema-drift check timed out after ${STEP_TIMEOUT}s." \ - "The database accepted the connection but never answered. This is NOT confirmed drift." + reason="the schema-drift check timed out after ${STEP_TIMEOUT}s; the database accepted the connection but never answered" else - fatal "FATAL: could not check for schema drift — prisma migrate diff exited $DRIFT_STATUS." \ - "The database was NEVER COMPARED. This is NOT confirmed drift." \ - "Causes, roughly by likelihood: DATABASE_URL unset or wrong; database unreachable" \ - "from this container; the pinned prisma CLI (apps/worker/Dockerfile) rejecting" \ - "schema.prisma; a missing or unloadable schema engine binary." + reason="prisma migrate diff exited $DRIFT_STATUS without comparing the database" fi + # The override is consulted BEFORE emitting FATAL. Calling fatal() first would + # page on every successfully-overridden boot, which is exactly what the + # stdout/stderr split above exists to avoid. if drift_override_set; then - echo "[worker] WARNING: OUTPOST_ALLOW_SCHEMA_DRIFT is set — starting unverified." >&2 + echo "[worker] WARNING: $reason." >&2 + echo "[worker] WARNING: this is NOT confirmed drift — OUTPOST_ALLOW_SCHEMA_DRIFT is set, starting unverified." >&2 + SCHEMA_VERIFIED=no else - echo "[worker] To boot without the guard, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." >&2 + fatal "FATAL: $reason." \ + "The database was NEVER COMPARED. This is NOT confirmed drift." \ + "Causes, roughly by likelihood: DATABASE_URL unset or wrong; database unreachable" \ + "from this container; the pinned prisma CLI (apps/worker/Dockerfile) rejecting" \ + "schema.prisma; a missing or unloadable schema engine binary." \ + "To boot without the guard, set OUTPOST_ALLOW_SCHEMA_DRIFT=1." exit 1 fi fi -echo "[worker] Migrations complete and schema verified. Starting worker..." +# The success line must not claim verification on either override path — one starts +# against known drift, the other against a database that was never compared. +if [ "$SCHEMA_VERIFIED" = yes ]; then + echo "[worker] Migrations complete and schema verified. Starting worker..." +else + echo "[worker] Migrations complete; schema NOT verified (override in effect). Starting worker..." +fi exec node apps/worker/dist/index.js diff --git a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql index 6d9d4615..c16d3695 100644 --- a/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql +++ b/packages/outpost/db/prisma/migrations/20260812220000_create_missing_systemconfig_table/migration.sql @@ -10,7 +10,7 @@ -- --to-schema-datamodel` against that database reported "No difference detected". -- That compares indexes, constraints and column types, not just tables — so -- SystemConfig really was the only difference, and the rest of 0001_init (which --- creates 45 indexes and 12 foreign keys AFTER this table) did land. Staging was +-- creates 27 indexes and 12 foreign keys AFTER this table) did land. Staging was -- not checked; its Postgres has no public URL, and the drift guard in -- apps/worker/start.sh is what will report the answer on its next deploy. -- @@ -53,15 +53,41 @@ CREATE TABLE IF NOT EXISTS "SystemConfig" ( -- services and demands a human. That is deliberately louder than the alternative: -- a stopped deploy is recoverable, a silently-ineffective one cost nine days. DO $$ +DECLARE + rel oid := to_regclass(format('%I.%I', current_schema(), 'SystemConfig')); BEGIN - IF ( - SELECT count(*) - FROM information_schema.columns - WHERE table_schema = 'public' - AND table_name = 'SystemConfig' - AND column_name IN ('key', 'value', 'updatedAt') - ) <> 3 THEN + -- pg_catalog, not information_schema: the latter only exposes columns the + -- current role holds some privilege on. This table was created directly in + -- production, possibly as another role, so an information_schema count could + -- return 0 against a perfectly correct table and hard-fail the migration into + -- P3009 — blocking both services over a database that was fine. + -- + -- current_schema(), not a hardcoded 'public': the CREATE above is unqualified + -- and resolves through search_path, so an environment using ?schema= would + -- create the table in one schema while this inspected another. + IF rel IS NULL THEN RAISE EXCEPTION - 'SystemConfig exists with unexpected columns; repairing it needs an ALTER, not this migration. Compare the live table against the SystemConfig model in schema.prisma.'; + 'SystemConfig does not exist in schema % after CREATE TABLE IF NOT EXISTS; the migration could not repair it.', current_schema(); + END IF; + + -- Shape, not just names. Three correctly-named columns of the wrong type, or + -- nullable where the schema says NOT NULL, or an extra column, or a missing + -- primary key (Prisma's upsert on @id key needs it) all leave a database that + -- does not match schema.prisma while CREATE ... IF NOT EXISTS quietly no-ops. + -- Letting any of those record as applied is the "recorded but not effective" + -- state this migration exists to eliminate, reproduced one level up. + IF NOT ( + (SELECT count(*) FROM pg_attribute + WHERE attrelid = rel AND attnum > 0 AND NOT attisdropped) = 3 + AND EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = rel + AND attname = 'key' AND atttypid = 'text'::regtype AND attnotnull) + AND EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = rel + AND attname = 'value' AND atttypid = 'text'::regtype AND attnotnull) + AND EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = rel + AND attname = 'updatedAt' AND atttypid = 'timestamp'::regtype AND attnotnull) + AND EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = rel AND contype = 'p') + ) THEN + RAISE EXCEPTION + 'SystemConfig exists but does not match schema.prisma (columns, types, nullability, or primary key). Repairing it needs an ALTER, not this migration. Compare the live table against the SystemConfig model in schema.prisma.'; END IF; END $$; From 36b263b37ef783e916d250940ec738ebcc5cc1ee Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 10:59:30 -0700 Subject: [PATCH 78/83] ci: supply PGPASSWORD to the migration-verification step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The idempotency fix added a psql call to the first verification step, but PGPASSWORD was only set on the second — so the step failed with 'no password supplied' before reaching anything it was meant to assert. Also pins psql to 127.0.0.1: the runner resolved localhost to ::1. --- .github/workflows/ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bcc0c81..30ff54fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,8 @@ jobs: - name: Verify Prisma schema and migrations env: - DATABASE_URL: 'postgresql://ci:ci@localhost:5432/ci' + DATABASE_URL: 'postgresql://ci:ci@127.0.0.1:5432/ci' + PGPASSWORD: ci run: | set -e pnpm --filter @copilotkit/outpost exec prisma validate --schema db/prisma/schema.prisma @@ -98,14 +99,14 @@ jobs: # blind spot this whole job exists to close. To actually exercise # CREATE TABLE IF NOT EXISTS, the row has to be removed while the # correctly-shaped table stays. - psql -h localhost -U ci -d ci -q -c \ + psql -h 127.0.0.1 -U ci -d ci -q -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma echo "Repair migration re-runs cleanly against an already-correct table." - name: Verify the SystemConfig repair path env: - DATABASE_URL: 'postgresql://ci:ci@localhost:5432/ci' + DATABASE_URL: 'postgresql://ci:ci@127.0.0.1:5432/ci' PGPASSWORD: ci run: | set -e @@ -114,8 +115,8 @@ jobs: # case — production had 0001_init RECORDED AS APPLIED while the table it # declares was absent, and that is the one scenario this migration exists # for. Without this, nothing tests it. - psql -h localhost -U ci -d ci -c 'DROP TABLE IF EXISTS "SystemConfig";' - psql -h localhost -U ci -d ci -c \ + psql -h 127.0.0.1 -U ci -d ci -c 'DROP TABLE IF EXISTS "SystemConfig";' + psql -h 127.0.0.1 -U ci -d ci -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" # The database now looks exactly like production did on 2026-08-07: @@ -153,12 +154,12 @@ jobs: # A table present but mis-shaped must FAIL the migration loudly rather # than no-op and be recorded as applied — the "recorded but not # effective" state that caused the incident. - psql -h localhost -U ci -d ci -c 'DROP TABLE "SystemConfig";' + psql -h 127.0.0.1 -U ci -d ci -c 'DROP TABLE "SystemConfig";' # Wrong TYPE, not just a missing column name — the assertion must check # shape, since a table with the right three names but wrong types is # still a database that does not match schema.prisma. - psql -h localhost -U ci -d ci -c 'CREATE TABLE "SystemConfig" ("key" TEXT PRIMARY KEY, "value" INTEGER NOT NULL, "updatedAt" TIMESTAMP(3) NOT NULL);' - psql -h localhost -U ci -d ci -c \ + psql -h 127.0.0.1 -U ci -d ci -c 'CREATE TABLE "SystemConfig" ("key" TEXT PRIMARY KEY, "value" INTEGER NOT NULL, "updatedAt" TIMESTAMP(3) NOT NULL);' + psql -h 127.0.0.1 -U ci -d ci -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" set +e deploy_out=$(pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma 2>&1) @@ -184,7 +185,7 @@ jobs: # (mis-shaped table plus a failed P3009 row), and they run before Build # and Test — so the first DB-touching test added to this job would # otherwise inherit a knowingly-broken schema. - psql -h localhost -U ci -d ci -q -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' + psql -h 127.0.0.1 -U ci -d ci -q -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma echo "CI database restored." From f2673caa87c1f5f7633b2f0c5900dc611265574e Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 11:01:48 -0700 Subject: [PATCH 79/83] ci: invoke prisma directly so exit codes survive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pnpm exec remaps any non-zero child status to 1 (ERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL), which collapses migrate diff's 0/2/other contract into pass/fail. The previous run proved it: the guard correctly reported real drift as exit 2 and the assertion saw 1, failing with 'it never compared the database' when it had. apps/worker/start.sh was never affected — it calls the CLI directly — but the CI that verifies that guard has to invoke it the same way. Adds a positive control asserting a deliberate exit 2 arrives as 2. --- .github/workflows/ci.yml | 50 +++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30ff54fd..99a129d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,14 @@ jobs: PGPASSWORD: ci run: | set -e - pnpm --filter @copilotkit/outpost exec prisma validate --schema db/prisma/schema.prisma + # Invoke the CLI directly, the way apps/worker/start.sh does. `pnpm exec` + # remaps ANY non-zero child status to 1 (ERR_PNPM_RECURSIVE_EXEC_FIRST_FAIL), + # which destroys the 0/2/other contract these assertions depend on — the + # first run of this job proved it, reporting real drift (exit 2) as a + # tool failure (exit 1). + PRISMA="node $(cd packages/outpost && node -p "require.resolve('prisma/build/index.js')")" + echo "using $PRISMA" + $PRISMA validate --schema packages/outpost/db/prisma/schema.prisma test -f packages/outpost/db/prisma/migrations/migration_lock.toml # Apply every migration to a real, empty database, then ask whether the @@ -86,10 +93,10 @@ jobs: # failures in August 2026. Running the same `migrate diff` the worker's # start.sh uses closes the loop: if the migrations cannot reproduce # schema.prisma from scratch, this fails here instead of at 3am. - pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma - pnpm --filter @copilotkit/outpost exec prisma migrate diff \ - --from-schema-datasource db/prisma/schema.prisma \ - --to-schema-datamodel db/prisma/schema.prisma \ + $PRISMA migrate deploy --schema packages/outpost/db/prisma/schema.prisma + $PRISMA migrate diff \ + --from-schema-datasource packages/outpost/db/prisma/schema.prisma \ + --to-schema-datamodel packages/outpost/db/prisma/schema.prisma \ --exit-code echo "Migrations reproduce schema.prisma exactly." @@ -101,15 +108,26 @@ jobs: # correctly-shaped table stays. psql -h 127.0.0.1 -U ci -d ci -q -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" - pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + $PRISMA migrate deploy --schema packages/outpost/db/prisma/schema.prisma echo "Repair migration re-runs cleanly against an already-correct table." + # Positive control on the thing every assertion below rests on: that + # this invocation style returns the child's exit code unchanged. + set +e + sh -c 'exit 2'; propagated=$? + set -e + if [ "$propagated" -ne 2 ]; then + echo "FATAL: exit codes are not propagating (got $propagated for a deliberate 2)." + exit 1 + fi + - name: Verify the SystemConfig repair path env: DATABASE_URL: 'postgresql://ci:ci@127.0.0.1:5432/ci' PGPASSWORD: ci run: | set -e + PRISMA="node $(cd packages/outpost && node -p "require.resolve('prisma/build/index.js')")" # Reconstruct the broken production state and prove the repair fixes it. # The step above only covers a fresh database, which was never the broken # case — production had 0001_init RECORDED AS APPLIED while the table it @@ -127,9 +145,9 @@ jobs: # as proof the guard works, having never compared anything. start.sh # treats that distinction as load-bearing; the test for it must too. set +e - pnpm --filter @copilotkit/outpost exec prisma migrate diff \ - --from-schema-datasource db/prisma/schema.prisma \ - --to-schema-datamodel db/prisma/schema.prisma \ + $PRISMA migrate diff \ + --from-schema-datasource packages/outpost/db/prisma/schema.prisma \ + --to-schema-datamodel packages/outpost/db/prisma/schema.prisma \ --exit-code drift_status=$? set -e @@ -144,10 +162,10 @@ jobs: echo "Drift guard detects the missing table (exit 2)." # ...and the repair migration must fix it. - pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma - pnpm --filter @copilotkit/outpost exec prisma migrate diff \ - --from-schema-datasource db/prisma/schema.prisma \ - --to-schema-datamodel db/prisma/schema.prisma \ + $PRISMA migrate deploy --schema packages/outpost/db/prisma/schema.prisma + $PRISMA migrate diff \ + --from-schema-datasource packages/outpost/db/prisma/schema.prisma \ + --to-schema-datamodel packages/outpost/db/prisma/schema.prisma \ --exit-code echo "Repair migration restores SystemConfig." @@ -162,7 +180,7 @@ jobs: psql -h 127.0.0.1 -U ci -d ci -c \ "DELETE FROM _prisma_migrations WHERE migration_name = '20260812220000_create_missing_systemconfig_table';" set +e - deploy_out=$(pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma 2>&1) + deploy_out=$($PRISMA migrate deploy --schema packages/outpost/db/prisma/schema.prisma 2>&1) deploy_status=$? set -e echo "$deploy_out" @@ -186,7 +204,7 @@ jobs: # and Test — so the first DB-touching test added to this job would # otherwise inherit a knowingly-broken schema. psql -h 127.0.0.1 -U ci -d ci -q -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' - pnpm --filter @copilotkit/outpost exec prisma migrate deploy --schema db/prisma/schema.prisma + $PRISMA migrate deploy --schema packages/outpost/db/prisma/schema.prisma echo "CI database restored." - name: Verify the worker startup guard @@ -243,7 +261,7 @@ jobs: # report false drift and block every deploy with a message pointing at # the database. Keep the Dockerfile pin and the lockfile in lockstep. set -o pipefail - resolved=$(pnpm --filter @copilotkit/outpost exec prisma --version | sed -n 's/^prisma *: *\([0-9][^ ]*\).*/\1/p' | head -1) + resolved=$(node "$(cd packages/outpost && node -p "require.resolve('prisma/build/index.js')")" --version | sed -n 's/^prisma *: *\([0-9][^ ]*\).*/\1/p' | head -1) if [ -z "$resolved" ]; then echo "FATAL: could not parse a version from 'prisma --version'; its output format may have changed." exit 1 From 90b61de4db125b23b265f2f10cda5ffdf717dcfb Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 11:03:36 -0700 Subject: [PATCH 80/83] ci: match the migration's actual RAISE text The mis-shaped-table assertion greps the deploy output to prove the failure came from the migration's own guard rather than, say, an unreachable database. The guard's message was reworded when it grew type/nullability/PK checks; the pattern was not. CI then reported a correctly-firing guard as 'failed, but not via the migration's own guard'. Anchors on a stable fragment now. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a129d4..19c6678c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,8 +190,12 @@ jobs: fi # Any non-zero would also cover "database unreachable", which would pass # this test without the assertion ever firing. Require our own message. + # Match a stable fragment of the migration's own RAISE, not the whole + # sentence: the previous run failed here because the assertion text was + # reworded and this pattern was not, so a correctly-firing guard read as + # "failed for the wrong reason". case "$deploy_out" in - *"SystemConfig exists with unexpected columns"*) ;; + *"Repairing it needs an ALTER"*) ;; *) echo "FATAL: deploy failed, but not via the migration's own guard." exit 1 From 08417d4142cd54147394f1a4dcf36e0b2fd745b8 Mon Sep 17 00:00:00 2001 From: Jerel John Velarde Date: Thu, 13 Aug 2026 11:06:03 -0700 Subject: [PATCH 81/83] ci: stop the guard harness aborting on expected failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The step runs under `bash -e`, and over half the guard cases are supposed to exit non-zero — so `out=$(...)` on the first blocked-boot case aborted the step before it could compare anything. The two stream checks had the same problem inverted: `... | grep -q FATAL && { exit 1; }` returns non-zero precisely when the assertion PASSES. Verified by extracting the block from the workflow and running it under bash -e: all eleven cases pass, exit 0. --- .github/workflows/ci.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19c6678c..047c098f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,7 +225,13 @@ jobs: check() { # label expect_exit expect_started env... label=$1 want_rc=$2 want_started=$3; shift 3 - out=$(env OUTPOST_PRISMA_CMD="$stub" "$@" sh "$script" 2>&1); rc=$? + # set +e around the call: half these cases are SUPPOSED to exit + # non-zero, and this step runs under `bash -e`, so the assignment + # would abort the whole step on the first expected failure. + set +e + out=$(env OUTPOST_PRISMA_CMD="$stub" "$@" sh "$script" 2>&1) + rc=$? + set -e case "$out" in *STARTED*) got=yes ;; *) got=no ;; esac if [ "$rc" != "$want_rc" ] || [ "$got" != "$want_started" ]; then echo "FATAL: $label -> exit=$rc started=$got (wanted exit=$want_rc started=$want_started)" @@ -251,10 +257,20 @@ jobs: check "timeout=0 means unbounded" 0 yes DIFF_RC=0 OUTPOST_STARTUP_STEP_TIMEOUT=0 # Diagnostics must reach stderr, and an overridden boot must not page. - env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=2 sh "$script" 2>/dev/null | grep -q FATAL \ - && { echo "FATAL: diagnostics leaked to stdout."; exit 1; } - env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=1 OUTPOST_ALLOW_SCHEMA_DRIFT=1 sh "$script" 2>&1 >/dev/null | grep -q FATAL \ - && { echo "FATAL: an overridden boot emitted FATAL and would page."; exit 1; } + set +e + env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=2 sh "$script" 2>/dev/null | grep -q FATAL + leaked=$? + env OUTPOST_PRISMA_CMD="$stub" DIFF_RC=1 OUTPOST_ALLOW_SCHEMA_DRIFT=1 sh "$script" 2>&1 >/dev/null | grep -q FATAL + paged=$? + set -e + if [ "$leaked" -eq 0 ]; then + echo "FATAL: diagnostics leaked to stdout." + exit 1 + fi + if [ "$paged" -eq 0 ]; then + echo "FATAL: an overridden boot emitted FATAL and would page." + exit 1 + fi echo "Startup guard behaves correctly across all branches." - name: Verify the pinned Prisma CLI matches the lockfile From 34f4ba9f45e2ee3d603cb6019905379f684bbb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:15:30 -0400 Subject: [PATCH 82/83] feat(queue): make one-answer-per-ticket hold under concurrency and delivery failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilt from main rather than rebased. PR #170 carried both the customer-facing fix and this infrastructure work; #172 took the fix and shipped, so what remains is the half that was never reviewed. A rebase would have replayed 24 commits that predate the #180 hotfix and, as the two-dot diff showed, would have DELETED the SystemConfig repair migration, the schema-drift guard in start.sh, and its CI gate. Rebuilding on current main keeps all three. #172 shipped two layers of the rule: reply paths no longer enqueue, and a re-answer gate reads the ticket's history. Both are check-then-act against a snapshot, so neither survives two jobs running at once. Two ways that happens today, neither needing a duplicate inbound event: - runWithTimeout in worker.ts is a Promise.race that does not cancel the handler. AI_RESPONSE has a 120s timeout over a deliberately slow pipeline, so on timeout attempt 1 keeps running while attempt 2 starts a second later, reads messages before attempt 1's message.create, and both post. - AI_RESPONSE runs at concurrency 4. The database now arbitrates. Message gains a nullable responseKey with a unique index on (ticketId, responseKey); exactly one PRIMARY_AI_RESPONSE row can exist per ticket, and the row is claimed BEFORE any platform post-back, so the loser catches a narrowly-identified P2002 and returns without posting. Nullable on purpose: historical BOT rows stay valid and still prove a ticket was answered. Delivery is a state machine rather than a hope. responseState is PENDING -> DELIVERED | ESCALATED, every transition a compare-and-set via updateMany guarded on responseKey and current state, never a blind update by id. A response starts PENDING before any external post; successful delivery marks DELIVERED; one needing a human stays PENDING until that escalation is durable, then becomes ESCALATED. If delivery itself ends PENDING, a retry schedules a delayed check that pulls in a human only if the response is still pending, so the one-post rule holds without racing the original handler. Job gains claimToken so a timed-out attempt cannot write back over the attempt that replaced it. Stale PROCESSING jobs are reclaimed after a grace period, and graceful shutdown shares one drain promise so signal handlers and the app cannot each half-drain. Also here: - generator.ts read only content[0], silently discarding text from multi-block model responses, and now concatenates all text blocks in order. It also throws on empty text rather than publishing nothing — unrequested scope, filed as #178 for an explicit decision. - Postmark ticket creation is idempotent, backed by a partial unique index on Ticket(sourceId) WHERE source = 'EMAIL'. Reply detection falls back to In-Reply-To and References so a reply without a plus-address stops becoming a second ticket with its own answer. - The Teams acknowledgement card no longer claims an AI is reviewing the question when no AI job was enqueued. Restores the /health rework that #180 deferred (d0c6f99). #180's own start.sh guard catches the SystemConfig cause; it does not catch the class. Any other boot-time failure — unreachable database, bad credentials, a throw inside buildSyncEngine unrelated to schema — still exits before the port binds, leaving Railway nothing to report but a five-minute timeout indistinguishable from a broken image. The port now binds first, boot state is tracked, and /health answers 503 with the phase and reason while boot is unfinished or failed. Fail-fast is retained: an unbooted worker is never reported healthy, because its sync mappings come from the database and one running against a schema it does not match would write wrong statuses to Linear. railway.toml drops healthcheckTimeout from Railway's 300s default to 180s, above BOOT_FAILURE_LINGER_MS, so the probe reads the reason before the process exits to be restarted. This is the reviewed version of that work recovered from d0c6f99~1, not a rewrite — it carries the review rounds that made the boot-failure path reachable and shutdown survive a signal during boot. Worker tests go 2 -> 24. Verification: turbo typecheck 10/10, turbo test 10/10 (1924 tests). Confirmed by diff that start.sh, both Dockerfiles, .github/workflows/ci.yml, .env.example and the 20260812 SystemConfig migration are byte-identical to main. Three findings from the split brief are NOT fixed here and block merge: escalationEnqueued is assigned and never read in the non-recovery path; the In-Reply-To/References resolution trusts attacker-controlled headers, so anyone holding a Message-ID from a thread can append to that ticket; and responseError carries DELIVERY_CONFIRMED / ESCALATION_REQUIRED control state in a free-text error column, in the same change that introduced an enum for exactly that. --- apps/teams-bot/src/__tests__/cards.test.ts | 7 +- .../src/cards/ticket-created-card.ts | 8 +- apps/teams-bot/src/handlers/message.ts | 1 + .../src/__tests__/postmark-webhook.test.ts | 531 +++++++++++++- .../src/app/api/webhooks/postmark/route.ts | 286 ++++++-- .../src/app/api/webhooks/postmark/utils.ts | 66 ++ apps/worker/railway.toml | 6 + apps/worker/src/__tests__/health.test.ts | 280 ++++++++ apps/worker/src/health.ts | 213 ++++++ apps/worker/src/index.ts | 363 ++++++++-- packages/outpost/ai/src/generator.test.ts | 25 + packages/outpost/ai/src/generator.ts | 11 +- .../migration.sql | 7 + .../migration.sql | 6 + .../migration.sql | 2 + .../migration.sql | 6 + packages/outpost/db/prisma/schema.prisma | 12 + .../queue/src/__tests__/ai-response.test.ts | 661 +++++++++++++++++- .../outpost/queue/src/__tests__/queue.test.ts | 255 ++++++- .../src/__tests__/worker-concurrency.test.ts | 40 +- packages/outpost/queue/src/create-job.ts | 12 +- .../outpost/queue/src/handlers/ai-response.ts | 513 ++++++++++++-- packages/outpost/queue/src/types.ts | 7 + packages/outpost/queue/src/worker.ts | 246 +++++-- 24 files changed, 3215 insertions(+), 349 deletions(-) create mode 100644 apps/worker/src/__tests__/health.test.ts create mode 100644 apps/worker/src/health.ts create mode 100644 packages/outpost/db/prisma/migrations/20260811193000_add_message_response_key/migration.sql create mode 100644 packages/outpost/db/prisma/migrations/20260811200000_add_message_response_state/migration.sql create mode 100644 packages/outpost/db/prisma/migrations/20260811220000_add_job_claim_token/migration.sql create mode 100644 packages/outpost/db/prisma/migrations/20260811230000_add_unique_postmark_message_id/migration.sql diff --git a/apps/teams-bot/src/__tests__/cards.test.ts b/apps/teams-bot/src/__tests__/cards.test.ts index ad7a767e..143a90c3 100644 --- a/apps/teams-bot/src/__tests__/cards.test.ts +++ b/apps/teams-bot/src/__tests__/cards.test.ts @@ -153,6 +153,7 @@ describe('buildTicketCreatedCard', () => { it('builds a ticket acknowledgment card', () => { const card = buildTicketCreatedCard({ title: 'Help with integration', + aiJobEnqueued: true, }); expect(card.type).toBe('AdaptiveCard'); @@ -167,6 +168,7 @@ describe('buildTicketCreatedCard', () => { // property today is a no-op; the day the builder reads it, this fails. const card = buildTicketCreatedCard({ title: 'Help with integration', + aiJobEnqueued: true, ticketDisplayId: 'TKT-LEAK01', } as TicketCreatedCardOptions); @@ -179,7 +181,10 @@ describe('buildTicketCreatedCard', () => { // (handlers/message.ts passes truncate(message.content)), so it is // rendered as-is. Callers must never put an internal displayId here — // this builder does not sanitize, and this test pins that contract. - const card = buildTicketCreatedCard({ title: 'my ref is TKT-USERTYPED' }); + const card = buildTicketCreatedCard({ + title: 'my ref is TKT-USERTYPED', + aiJobEnqueued: true, + }); const body = card.body as Array<{ text: string }>; expect(body[1].text).toBe('my ref is TKT-USERTYPED'); diff --git a/apps/teams-bot/src/cards/ticket-created-card.ts b/apps/teams-bot/src/cards/ticket-created-card.ts index b7d19a3a..191ae3f6 100644 --- a/apps/teams-bot/src/cards/ticket-created-card.ts +++ b/apps/teams-bot/src/cards/ticket-created-card.ts @@ -1,5 +1,7 @@ export interface TicketCreatedCardOptions { title: string; + /** Whether an AI_RESPONSE job was actually queued for this ticket. */ + aiJobEnqueued: boolean; } /** @@ -9,7 +11,7 @@ export interface TicketCreatedCardOptions { * belongs in the dashboard and team slash commands, not in reporter-facing copy. */ export function buildTicketCreatedCard(options: TicketCreatedCardOptions): Record { - const { title } = options; + const { title, aiJobEnqueued } = options; return { type: 'AdaptiveCard', @@ -30,7 +32,9 @@ export function buildTicketCreatedCard(options: TicketCreatedCardOptions): Recor }, { type: 'TextBlock', - text: 'Our AI assistant is reviewing your question...', + text: aiJobEnqueued + ? 'Our AI assistant is reviewing your question...' + : 'A team member will review your question and follow up.', wrap: true, }, ], diff --git a/apps/teams-bot/src/handlers/message.ts b/apps/teams-bot/src/handlers/message.ts index c2bebc5d..a79e184a 100644 --- a/apps/teams-bot/src/handlers/message.ts +++ b/apps/teams-bot/src/handlers/message.ts @@ -101,6 +101,7 @@ export async function handleMessage(context: TurnContext): Promise { // other platforms did. const card = buildTicketCreatedCard({ title: truncate(message.content, 200), + aiJobEnqueued: result.aiJobEnqueued, }); const reply = MessageFactory.attachment( diff --git a/apps/web/src/__tests__/postmark-webhook.test.ts b/apps/web/src/__tests__/postmark-webhook.test.ts index cf8b10e3..aa90bc23 100644 --- a/apps/web/src/__tests__/postmark-webhook.test.ts +++ b/apps/web/src/__tests__/postmark-webhook.test.ts @@ -6,19 +6,29 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; // ─── Mock Prisma ──────────────────────────────────────────────────────────── const mockTicketFindUnique = vi.fn(); +const mockTicketFindFirst = vi.fn(); const mockTicketCreate = vi.fn(); const mockTicketUpdate = vi.fn(); const mockMessageCreate = vi.fn(); +const mockMessageFindFirst = vi.fn(); +const mockJobCreate = vi.fn(); +const mockTransaction = vi.fn(); vi.mock('@copilotkit/outpost/db', () => ({ prisma: { + $transaction: (...args: unknown[]) => mockTransaction(...args), ticket: { findUnique: (...args: unknown[]) => mockTicketFindUnique(...args), + findFirst: (...args: unknown[]) => mockTicketFindFirst(...args), create: (...args: unknown[]) => mockTicketCreate(...args), update: (...args: unknown[]) => mockTicketUpdate(...args), }, message: { create: (...args: unknown[]) => mockMessageCreate(...args), + findFirst: (...args: unknown[]) => mockMessageFindFirst(...args), + }, + job: { + create: (...args: unknown[]) => mockJobCreate(...args), }, }, })); @@ -45,7 +55,15 @@ vi.mock('@copilotkit/outpost/queue', () => ({ // ─── Import route + helpers ───────────────────────────────────────────────── import { POST } from '@/app/api/webhooks/postmark/route'; -import { extractTicketId, extractEmail, extractName } from '@/app/api/webhooks/postmark/utils'; +import { + extractTicketId, + extractEmail, + extractName, + extractReplyMessageIds, + getHeaderValue, + hasReplyHeaders, + normalizeMessageId, +} from '@/app/api/webhooks/postmark/utils'; import type { PostmarkInboundPayload } from '@/app/api/webhooks/postmark/utils'; // ─── Helpers ──────────────────────────────────────────────────────────────── @@ -76,6 +94,30 @@ function fullPayload(overrides: Partial = {}): PostmarkI describe('Postmark inbound webhook', () => { beforeEach(() => { vi.clearAllMocks(); + mockTicketCreate.mockReset(); + mockTicketFindFirst.mockReset(); + mockJobCreate.mockReset(); + mockTransaction.mockReset(); + mockCreateJob.mockReset(); + mockMessageFindFirst.mockReset(); + mockTicketFindUnique.mockReset(); + mockTicketFindFirst.mockResolvedValue(null); + mockMessageFindFirst.mockResolvedValue(null); + mockTicketFindUnique.mockResolvedValue(null); + mockJobCreate.mockResolvedValue({ id: 'job-1' }); + mockCreateJob.mockResolvedValue('job-1'); + mockTransaction.mockImplementation( + async ( + callback: (tx: { + ticket: { create: typeof mockTicketCreate }; + job: { create: typeof mockJobCreate }; + }) => Promise, + ) => + callback({ + ticket: { create: mockTicketCreate }, + job: { create: mockJobCreate }, + }), + ); }); // ── Helper function tests ────────────────────────────────────────────── @@ -126,6 +168,89 @@ describe('Postmark inbound webhook', () => { }); }); + // Postmark's MessageID field is bare while header values are angle-bracketed + // and may be folded across lines. If normalization is off by a bracket the + // route silently stops recognizing replies, so it is tested directly. + describe('normalizeMessageId', () => { + it('strips angle brackets', () => { + expect(normalizeMessageId('')).toBe('abc@example.com'); + }); + + it('leaves a bare ID untouched', () => { + expect(normalizeMessageId('abc@example.com')).toBe('abc@example.com'); + }); + + it('strips surrounding and inner-edge whitespace, including folded lines', () => { + expect(normalizeMessageId('\r\n\t ')).toBe('abc@example.com'); + expect(normalizeMessageId('< abc@example.com >')).toBe('abc@example.com'); + }); + + it('returns null for empty, bracket-only, and missing values', () => { + expect(normalizeMessageId('')).toBeNull(); + expect(normalizeMessageId(' ')).toBeNull(); + expect(normalizeMessageId('<>')).toBeNull(); + expect(normalizeMessageId(undefined)).toBeNull(); + expect(normalizeMessageId(null)).toBeNull(); + }); + }); + + describe('getHeaderValue', () => { + it('matches header names case-insensitively', () => { + const list = [{ Name: 'in-REPLY-to', Value: '' }]; + expect(getHeaderValue(list, 'In-Reply-To')).toBe(''); + }); + + it('returns undefined for a missing header or missing list', () => { + expect(getHeaderValue([{ Name: 'Date', Value: 'x' }], 'References')).toBeUndefined(); + expect(getHeaderValue(undefined, 'References')).toBeUndefined(); + }); + }); + + describe('extractReplyMessageIds', () => { + it('collects In-Reply-To and the whole References chain, normalized and deduped', () => { + expect( + extractReplyMessageIds([ + { Name: 'In-Reply-To', Value: '' }, + { Name: 'References', Value: ' \r\n\t' }, + ]), + ).toEqual(['b@x', 'a@x', 'c@x']); + }); + + it('tolerates comma-separated References', () => { + expect(extractReplyMessageIds([{ Name: 'References', Value: ', ' }])).toEqual( + ['a@x', 'b@x'], + ); + }); + + it('returns an empty list when there are no threading headers', () => { + expect(extractReplyMessageIds([{ Name: 'Subject', Value: 'hi' }])).toEqual([]); + expect(extractReplyMessageIds(undefined)).toEqual([]); + }); + + it('drops unparseable tokens', () => { + expect(extractReplyMessageIds([{ Name: 'In-Reply-To', Value: '<>' }])).toEqual([]); + }); + }); + + describe('hasReplyHeaders', () => { + it('is true for a non-empty In-Reply-To or References', () => { + expect(hasReplyHeaders([{ Name: 'In-Reply-To', Value: '' }])).toBe(true); + expect(hasReplyHeaders([{ Name: 'References', Value: '' }])).toBe(true); + }); + + it('is true even when the value cannot be parsed into an ID', () => { + // A malformed threading header is still proof this is a reply, so the + // bot must stay silent rather than answering mid-conversation. + expect(hasReplyHeaders([{ Name: 'In-Reply-To', Value: '<>' }])).toBe(true); + }); + + it('is false for whitespace-only, absent, and undefined headers', () => { + expect(hasReplyHeaders([{ Name: 'References', Value: ' ' }])).toBe(false); + expect(hasReplyHeaders([{ Name: 'Subject', Value: 'hi' }])).toBe(false); + expect(hasReplyHeaders(undefined)).toBe(false); + }); + }); + // ── Route handler tests ──────────────────────────────────────────────── describe('POST handler', () => { @@ -154,11 +279,112 @@ describe('Postmark inbound webhook', () => { }), ); - // Should enqueue AI_RESPONSE job - expect(mockCreateJob).toHaveBeenCalledWith( - 'AI_RESPONSE', - { ticketId: 'ticket-1', source: 'web' }, + // Ticket, opening message, and AI job share one transaction. + expect(mockTransaction).toHaveBeenCalledTimes(1); + expect(mockJobCreate).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: 'AI_RESPONSE', + payload: { ticketId: 'ticket-1', source: 'web' }, + }), + }); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('creates one ticket and AI job for concurrent deliveries of the same MessageID', async () => { + const ticket = { + id: 'ticket-concurrent', + displayId: 'TKT-TESTID01', + source: 'EMAIL', + sourceId: 'msg-001@postmark.example', + }; + mockTicketFindFirst + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null) + .mockResolvedValue(ticket); + + let sourceIdClaimed = false; + mockTicketCreate.mockImplementation(async () => { + if (sourceIdClaimed) { + throw { + code: 'P2002', + meta: { target: 'Ticket_email_sourceId_key' }, + }; + } + sourceIdClaimed = true; + return ticket; + }); + + const [first, second] = await Promise.all([ + POST(postmarkRequest(fullPayload())), + POST(postmarkRequest(fullPayload())), + ]); + + expect(first.status).toBe(200); + expect(second.status).toBe(200); + expect(await first.json()).toMatchObject({ ticketId: 'TKT-TESTID01' }); + expect(await second.json()).toMatchObject({ ticketId: 'TKT-TESTID01' }); + expect(mockTicketCreate).toHaveBeenCalledTimes(2); + expect(mockJobCreate).toHaveBeenCalledTimes(1); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('rolls back ticket creation when the atomic AI job insert fails, then retries once', async () => { + const ticket = { + id: 'ticket-after-retry', + displayId: 'TKT-TESTID01', + source: 'EMAIL', + sourceId: 'msg-001@postmark.example', + }; + let committedTicket: typeof ticket | null = null; + let committedJobs = 0; + let ticketAttempts = 0; + let failJobInsert = true; + + mockTicketFindFirst.mockImplementation(async () => committedTicket); + mockTransaction.mockImplementation( + async ( + callback: (tx: { + ticket: { create: () => Promise }; + job: { create: () => Promise<{ id: string }> }; + }) => Promise, + ) => { + let stagedTicket: typeof ticket | null = null; + let stagedJob = false; + const result = await callback({ + ticket: { + create: async () => { + ticketAttempts += 1; + stagedTicket = ticket; + return ticket; + }, + }, + job: { + create: async () => { + if (failJobInsert) { + failJobInsert = false; + throw new Error('queue insert unavailable'); + } + stagedJob = true; + return { id: 'job-after-retry' }; + }, + }, + }); + committedTicket = stagedTicket; + if (stagedJob) committedJobs += 1; + return result; + }, ); + + const first = await POST(postmarkRequest(fullPayload())); + const retry = await POST(postmarkRequest(fullPayload())); + + expect(first.status).toBe(500); + expect(retry.status).toBe(200); + expect(ticketAttempts).toBe(2); + expect(committedTicket).toEqual(ticket); + expect(committedJobs).toBe(1); + expect(mockTransaction).toHaveBeenCalledTimes(2); + expect(mockCreateJob).not.toHaveBeenCalled(); }); it('appends to existing ticket via MailboxHash (plus-addressing)', async () => { @@ -296,7 +522,7 @@ describe('Postmark inbound webhook', () => { }, ); - it('creates new ticket when MailboxHash ticket is not found', async () => { + it('files an orphaned reply without enqueueing an AI response', async () => { mockTicketFindUnique.mockResolvedValue(null); mockTicketCreate.mockResolvedValue({ id: 'new-ticket', @@ -314,6 +540,290 @@ describe('Postmark inbound webhook', () => { expect(res.status).toBe(200); const body = await res.json(); expect(body.status).toBe('ticket_created'); + expect(mockTicketCreate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + description: 'I have a question about my invoice.', + messages: expect.objectContaining({ + create: expect.objectContaining({ + content: 'I have a question about my invoice.', + type: 'USER', + }), + }), + }), + }), + ); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + // ── Reply detection via RFC 5322 threading headers ────────────────── + // + // MailboxHash only survives when the customer's client preserves the + // plus-address. The normal case is a reply to a plain From address, which + // carries In-Reply-To / References and nothing else. Those replies used to + // fall through to the new-ticket branch and got a second AI answer for a + // conversation already in progress. + + /** Build a Postmark Headers array for a reply. */ + function headers(inReplyTo?: string, references?: string) { + const list = [ + { Name: 'Date', Value: 'Mon, 3 Feb 2025 10:00:00 +0000' }, + { Name: 'Subject', Value: 'Re: Need help with billing' }, + ]; + if (inReplyTo !== undefined) list.push({ Name: 'In-Reply-To', Value: inReplyTo }); + if (references !== undefined) list.push({ Name: 'References', Value: references }); + return list; + } + + /** + * Answer the reply-resolution ticket lookup (`sourceId: { in: [...] }`) + * from a map, while leaving the MessageID idempotency lookup + * (`sourceId: ''`) returning null. + */ + function ticketsBySourceId(map: Record) { + mockTicketFindFirst.mockImplementation(async (args: unknown) => { + const where = (args as { where?: { sourceId?: { in?: string[] } } }).where; + const ids = where?.sourceId?.in; + if (!Array.isArray(ids)) return null; + for (const id of ids) { + if (map[id]) return map[id]; + } + return null; + }); + } + + it('appends a reply whose In-Reply-To matches a ticket sourceId, with no AI job', async () => { + ticketsBySourceId({ + 'root-msg@postmark.example': { + id: 'ticket-root', + displayId: 'TKT-ROOT0001', + status: 'OPEN', + }, + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + MessageID: 'reply-msg@postmark.example', + Subject: 'Re: Need help with billing', + TextBody: 'Any update on this?', + Headers: headers(''), + }), + ), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + status: 'message_appended', + ticketId: 'TKT-ROOT0001', + }); + expect(mockMessageCreate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + ticketId: 'ticket-root', + content: 'Any update on this?', + // Every appended message records its inbound Message-ID so a + // later reply can resolve to this mid-thread message. + attachments: expect.objectContaining({ + postmarkMessageId: 'reply-msg@postmark.example', + }), + }), + }), + ); + // The whole point: no second ticket, no second answer. + expect(mockTicketCreate).not.toHaveBeenCalled(); + expect(mockTransaction).not.toHaveBeenCalled(); + expect(mockJobCreate).not.toHaveBeenCalled(); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('resolves a reply through the References chain when In-Reply-To names an outbound ID we never stored', async () => { + // Outbound Message-IDs are not persisted (postResponse is unimplemented), + // so a reply to our own message names an ID no row holds. References + // still carries the customer's opening Message-ID. + ticketsBySourceId({ + 'root-msg@postmark.example': { + id: 'ticket-root', + displayId: 'TKT-ROOT0001', + status: 'WAITING_ON_CUSTOMER', + }, + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + mockTicketUpdate.mockResolvedValue({}); + + const res = await POST( + postmarkRequest( + fullPayload({ + MessageID: 'reply-msg@postmark.example', + Headers: headers( + '', + '\r\n\t', + ), + }), + ), + ); + + expect(await res.json()).toMatchObject({ + status: 'message_appended', + ticketId: 'TKT-ROOT0001', + }); + // Dormant ticket reopens so a human sees the reply. + expect(mockTicketUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + where: { id: 'ticket-root' }, + data: expect.objectContaining({ status: 'OPEN' }), + }), + ); + expect(mockJobCreate).not.toHaveBeenCalled(); + }); + + it('resolves a reply that matches a mid-thread Message.attachments.postmarkMessageId', async () => { + ticketsBySourceId({}); + mockMessageFindFirst.mockResolvedValue({ + ticket: { id: 'ticket-mid', displayId: 'TKT-MID00001', status: 'OPEN' }, + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + MessageID: 'reply-msg@postmark.example', + Headers: headers(''), + }), + ), + ); + + expect(await res.json()).toMatchObject({ + status: 'message_appended', + ticketId: 'TKT-MID00001', + }); + expect(mockMessageFindFirst).toHaveBeenCalledWith( + expect.objectContaining({ + where: expect.objectContaining({ + OR: [ + { + attachments: { + path: ['postmarkMessageId'], + equals: 'mid-thread@postmark.example', + }, + }, + ], + }), + }), + ); + expect(mockTicketCreate).not.toHaveBeenCalled(); + expect(mockJobCreate).not.toHaveBeenCalled(); + }); + + it('files a reply whose headers resolve to nothing as a ticket with no AI job', async () => { + ticketsBySourceId({}); + mockTicketCreate.mockResolvedValue({ + id: 'ticket-orphan-header', + displayId: 'TKT-TESTID01', + }); + + const res = await POST( + postmarkRequest( + fullPayload({ + MessageID: 'reply-msg@postmark.example', + TextBody: 'Following up on the thread from last year.', + Headers: headers(''), + }), + ), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ status: 'ticket_created' }); + // Kept for a human... + expect(mockTicketCreate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + description: 'Following up on the thread from last year.', + }), + }), + ); + // ...but never answered. + expect(mockJobCreate).not.toHaveBeenCalled(); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('still answers a genuinely new email that carries headers but no threading headers', async () => { + mockTicketCreate.mockResolvedValue({ id: 'ticket-new', displayId: 'TKT-TESTID01' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + Headers: [ + { Name: 'Date', Value: 'Mon, 3 Feb 2025 10:00:00 +0000' }, + { Name: 'Subject', Value: 'Need help with billing' }, + { Name: 'Message-ID', Value: '' }, + ], + }), + ), + ); + + expect(res.status).toBe(200); + expect(await res.json()).toMatchObject({ status: 'ticket_created' }); + // The fix must not blanket-mute email: a new ticket still gets its one job. + expect(mockJobCreate).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: 'AI_RESPONSE', + payload: { ticketId: 'ticket-new', source: 'web' }, + }), + }); + // No threading headers means no header lookups at all. + expect(mockMessageFindFirst).not.toHaveBeenCalled(); + }); + + it('treats an empty References header as not-a-reply', async () => { + mockTicketCreate.mockResolvedValue({ id: 'ticket-new', displayId: 'TKT-TESTID01' }); + + await POST(postmarkRequest(fullPayload({ Headers: headers(undefined, ' ') }))); + + expect(mockJobCreate).toHaveBeenCalledTimes(1); + }); + + it('keeps MailboxHash as the primary reply path, without header lookups', async () => { + mockTicketFindUnique.mockResolvedValue({ + id: 'hash-ticket', + displayId: 'TKT-HASH0001', + status: 'OPEN', + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + MailboxHash: 'TKT-HASH0001', + Headers: headers(''), + }), + ), + ); + + expect(await res.json()).toMatchObject({ ticketId: 'TKT-HASH0001' }); + expect(mockTicketFindFirst).not.toHaveBeenCalled(); + expect(mockMessageFindFirst).not.toHaveBeenCalled(); + }); + + it('files an unresolvable MailboxHash reply without falling back to headers', async () => { + mockTicketFindUnique.mockResolvedValue(null); + mockTicketCreate.mockResolvedValue({ + id: 'ticket-orphan-hash', + displayId: 'TKT-TESTID01', + }); + + await POST( + postmarkRequest( + fullPayload({ + MailboxHash: 'TKT-NOTEXIST', + Headers: headers(''), + }), + ), + ); + + expect(mockMessageFindFirst).not.toHaveBeenCalled(); + expect(mockJobCreate).not.toHaveBeenCalled(); }); it('handles attachments in the payload', async () => { @@ -385,6 +895,15 @@ describe('Postmark inbound webhook', () => { expect(res.status).toBe(400); }); + it('rejects a new-email payload without the MessageID required for idempotency', async () => { + const res = await POST( + postmarkRequest(fullPayload({ MessageID: '' })), + ); + + expect(res.status).toBe(400); + expect(mockTransaction).not.toHaveBeenCalled(); + }); + it('returns 500 on database error', async () => { mockTicketCreate.mockRejectedValue(new Error('DB connection failed')); const consoleSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); diff --git a/apps/web/src/app/api/webhooks/postmark/route.ts b/apps/web/src/app/api/webhooks/postmark/route.ts index 37d9d99f..572cc65e 100644 --- a/apps/web/src/app/api/webhooks/postmark/route.ts +++ b/apps/web/src/app/api/webhooks/postmark/route.ts @@ -6,15 +6,144 @@ * - From, To, Subject, TextBody, HtmlBody, StrippedTextReply * - MailboxHash (plus-addressing: ticket+TKT-1234 -> TKT-1234) * - Headers, Attachments, MessageID + * + * One answer per ticket: a genuinely NEW email opens a ticket and gets exactly + * one AI response; a REPLY is appended to its existing ticket and gets none. + * Replies are detected first by `MailboxHash` (an exact ticket reference) and + * then by the RFC 5322 threading headers `In-Reply-To` / `References`, which is + * the only signal available when the customer's mail client replies to a plain + * From address and drops the plus-address. */ import crypto from 'node:crypto'; import { NextResponse } from 'next/server'; import { prisma } from '@copilotkit/outpost/db'; -import { generateTicketId, reopensOnCustomerReply } from '@copilotkit/outpost/shared'; -import { createJob, JobType } from '@copilotkit/outpost/queue'; -import { extractTicketId, extractEmail, extractName } from './utils'; +import { + generateTicketId, + MAX_JOB_ATTEMPTS, + reopensOnCustomerReply, +} from '@copilotkit/outpost/shared'; +import { JobType } from '@copilotkit/outpost/queue'; +import { + extractTicketId, + extractEmail, + extractName, + extractReplyMessageIds, + hasReplyHeaders, +} from './utils'; import type { PostmarkInboundPayload } from './utils'; +/** Ticket fields the reply paths need. */ +type ReplyTargetTicket = { id: string; displayId: string; status: string }; + +/** + * Resolve a reply to the ticket that already holds its conversation. + * + * Two places carry inbound Message-IDs: `Ticket.sourceId` (the email that + * OPENED the ticket) and `Message.attachments.postmarkMessageId` (every + * appended message). A reply can name either, so both are checked. Outbound + * Message-IDs are never persisted, which is why `References` (the full chain, + * including the customer's own opening ID) matters as much as `In-Reply-To`. + */ +async function findTicketByReplyMessageIds( + messageIds: string[], +): Promise { + if (messageIds.length === 0) return null; + + // The opening email of a thread — the oldest match wins so a thread always + // resolves to its root ticket. + const openingTicket = await prisma.ticket.findFirst({ + where: { source: 'EMAIL', sourceId: { in: messageIds } }, + orderBy: { createdAt: 'asc' }, + select: { id: true, displayId: true, status: true }, + }); + if (openingTicket) return openingTicket; + + // A message appended mid-thread. + const appendedMessage = await prisma.message.findFirst({ + where: { + ticket: { source: 'EMAIL' }, + OR: messageIds.map((id) => ({ + attachments: { path: ['postmarkMessageId'], equals: id }, + })), + }, + orderBy: { createdAt: 'asc' }, + select: { ticket: { select: { id: true, displayId: true, status: true } } }, + }); + return appendedMessage?.ticket ?? null; +} + +function isUniqueConstraintError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error as { code?: unknown }).code === 'P2002' + ); +} + +function ticketCreatedResponse(ticket: { displayId: string }) { + return NextResponse.json({ status: 'ticket_created', ticketId: ticket.displayId }); +} + +/** + * Record the inbound Message-ID on every message, not just ones with files. + * It is the only handle a later reply has on a mid-thread message, so it is + * persisted unconditionally. + */ +function buildMessageAttachments(body: PostmarkInboundPayload) { + return { + postmarkMessageId: body.MessageID, + ...(body.Attachments?.length + ? { + files: body.Attachments.map((a) => ({ + name: a.Name, + contentType: a.ContentType, + size: a.ContentLength, + })), + } + : {}), + }; +} + +/** + * Append a customer reply to the ticket that already owns the conversation. + * + * Shared by both reply paths (plus-address `MailboxHash` and RFC 5322 threading + * headers) so they cannot drift: same message write, same reopen rule, and — in + * both cases — no AI job. Outpost answers the opening email once and a human + * owns the rest of the thread. Not enqueuing here is what enforces that; the + * AI_RESPONSE handler's already-answered gate only backstops re-answering a + * ticket that already holds an AI response. + */ +async function appendReplyToTicket( + ticket: ReplyTargetTicket, + body: PostmarkInboundPayload, + author: string, + content: string, +) { + await prisma.message.create({ + data: { + ticketId: ticket.id, + author, + content, + type: 'USER', + attachments: buildMessageAttachments(body), + }, + }); + + // Re-open a dormant ticket so a human sees the reply. The status set lives + // in @copilotkit/outpost/shared so this path, the shared InboundHandler, and + // the GitHub App issue-comment webhook cannot drift apart. + if (reopensOnCustomerReply(ticket.status)) { + await prisma.ticket.update({ + where: { id: ticket.id }, + data: { status: 'OPEN', updatedAt: new Date() }, + }); + } + + return NextResponse.json({ status: 'message_appended', ticketId: ticket.displayId }); +} + export async function POST(request: Request) { // Webhook authentication via POSTMARK_WEBHOOK_TOKEN // Required in production; optional in development for local testing @@ -45,7 +174,7 @@ export async function POST(request: Request) { return NextResponse.json({ error: 'Invalid JSON' }, { status: 400 }); } - if (!body.From || !body.Subject) { + if (!body.From || !body.Subject || !body.MessageID) { return NextResponse.json({ error: 'Missing required fields' }, { status: 400 }); } @@ -54,91 +183,104 @@ export async function POST(request: Request) { const messageBody = body.StrippedTextReply || body.TextBody || ''; const ticketIdFromHash = extractTicketId(body.MailboxHash); + const author = `${senderName} <${senderEmail}>`; + try { - // If we have a ticket ID from plus-addressing, append to existing ticket + // MailboxHash is the primary reply path: it is an exact ticket reference + // and cheaper than header matching. if (ticketIdFromHash) { const existingTicket = await prisma.ticket.findUnique({ where: { displayId: ticketIdFromHash }, }); if (existingTicket) { - // Append message to existing ticket - await prisma.message.create({ + return await appendReplyToTicket(existingTicket, body, author, messageBody); + } + } else { + // Fallback: most mail clients reply to the plain From address and + // never preserve the plus-address, so RFC 5322 threading headers are + // the only thing marking those as replies. + const replyTarget = await findTicketByReplyMessageIds( + extractReplyMessageIds(body.Headers), + ); + if (replyTarget) { + return await appendReplyToTicket(replyTarget, body, author, messageBody); + } + } + + // A reply we could not resolve — a parsed MailboxHash or threading + // headers whose thread has no ticket (deleted ticket, or a thread that + // predates Outpost). Preserve the customer's words as a new + // ticket/message for a human, but do not spend an AI response on a + // mid-conversation message. + const isOrphanedReply = ticketIdFromHash !== null || hasReplyHeaders(body.Headers); + + // Postmark retries the same inbound delivery with the same MessageID. + // This read avoids deliberately colliding on the common retry path; the + // partial unique index remains the concurrency authority when two first + // deliveries pass this check together. + const existingInboundTicket = await prisma.ticket.findFirst({ + where: { source: 'EMAIL', sourceId: body.MessageID }, + select: { id: true, displayId: true }, + }); + if (existingInboundTicket) { + return ticketCreatedResponse(existingInboundTicket); + } + + // Create the ticket (including its nested opening Message) and its one + // AI_RESPONSE job in the same database transaction. A queue insert + // failure rolls the ticket back, so Postmark can retry from a clean + // state rather than creating a second ticket around a partial first run. + const displayId = generateTicketId(); + let ticket: { id: string; displayId: string }; + try { + ticket = await prisma.$transaction(async (tx) => { + const createdTicket = await tx.ticket.create({ data: { - ticketId: existingTicket.id, - author: `${senderName} <${senderEmail}>`, - content: messageBody, - type: 'USER', - attachments: body.Attachments?.length - ? { - postmarkMessageId: body.MessageID, - files: body.Attachments.map((a) => ({ - name: a.Name, - contentType: a.ContentType, - size: a.ContentLength, - })), - } - : undefined, + displayId, + title: body.Subject, + description: messageBody, + status: 'OPEN', + priority: 'MEDIUM', + type: 'QUESTION', + source: 'EMAIL', + sourceId: body.MessageID, + messages: { + create: { + author, + content: messageBody, + type: 'USER', + attachments: buildMessageAttachments(body), + }, + }, }, }); - // Re-open a dormant ticket so a human sees the reply. The status - // set lives in @copilotkit/outpost/shared so this path, the - // shared InboundHandler, and the GitHub App issue-comment - // webhook cannot drift apart. - if (reopensOnCustomerReply(existingTicket.status)) { - await prisma.ticket.update({ - where: { id: existingTicket.id }, - data: { status: 'OPEN', updatedAt: new Date() }, + // Only a genuinely new email gets the ticket's single AI response. + if (!isOrphanedReply) { + await tx.job.create({ + data: { + type: JobType.AI_RESPONSE, + payload: { ticketId: createdTicket.id, source: 'web' }, + maxAttempts: MAX_JOB_ATTEMPTS, + }, }); } - // No AI response on a reply — Outpost answers the opening email - // once and a human handles the rest of the thread. Not enqueuing - // here is what enforces that; the AI_RESPONSE handler's - // already-answered gate only backstops re-answering a ticket that - // already holds an AI response. - - return NextResponse.json({ status: 'message_appended', ticketId: existingTicket.displayId }); + return createdTicket; + }); + } catch (error) { + if (isUniqueConstraintError(error)) { + const concurrentTicket = await prisma.ticket.findFirst({ + where: { source: 'EMAIL', sourceId: body.MessageID }, + select: { id: true, displayId: true }, + }); + if (concurrentTicket) return ticketCreatedResponse(concurrentTicket); } + throw error; } - // Create new ticket from email - const displayId = generateTicketId(); - const ticket = await prisma.ticket.create({ - data: { - displayId, - title: body.Subject, - description: messageBody, - status: 'OPEN', - priority: 'MEDIUM', - type: 'QUESTION', - source: 'EMAIL', - sourceId: body.MessageID, - messages: { - create: { - author: `${senderName} <${senderEmail}>`, - content: messageBody, - type: 'USER', - attachments: body.Attachments?.length - ? { - postmarkMessageId: body.MessageID, - files: body.Attachments.map((a) => ({ - name: a.Name, - contentType: a.ContentType, - size: a.ContentLength, - })), - } - : undefined, - }, - }, - }, - }); - - // Enqueue AI response for the new ticket - await createJob(JobType.AI_RESPONSE, { ticketId: ticket.id, source: 'web' }); - - return NextResponse.json({ status: 'ticket_created', ticketId: ticket.displayId }); + return ticketCreatedResponse(ticket); } catch (err) { console.error('[Postmark Webhook] Error processing inbound email:', err); return NextResponse.json( diff --git a/apps/web/src/app/api/webhooks/postmark/utils.ts b/apps/web/src/app/api/webhooks/postmark/utils.ts index 88d56508..ce069bf8 100644 --- a/apps/web/src/app/api/webhooks/postmark/utils.ts +++ b/apps/web/src/app/api/webhooks/postmark/utils.ts @@ -21,6 +21,72 @@ export function extractName(from: string, fromName?: string): string { return match ? match[1].trim() : from.trim(); } +/** RFC 5322 threading headers that mark a message as a reply. */ +const REPLY_HEADERS = ['in-reply-to', 'references'] as const; + +/** Read a header value case-insensitively (SMTP header names are not case-sensitive). */ +export function getHeaderValue( + headers: Array<{ Name: string; Value: string }> | undefined, + name: string, +): string | undefined { + if (!headers?.length) return undefined; + const wanted = name.toLowerCase(); + return headers.find((h) => h?.Name?.toLowerCase() === wanted)?.Value; +} + +/** + * Normalize a Message-ID for comparison. + * + * Header values are angle-bracketed and may be folded across lines + * (`\r\n\t`), while Postmark's own `MessageID` field is bare. Both forms + * have to compare equal or a reply silently looks like a brand-new email. + */ +export function normalizeMessageId(raw: string | undefined | null): string | null { + if (!raw) return null; + const trimmed = raw.trim().replace(/^<+/, '').replace(/>+$/, '').trim(); + return trimmed.length > 0 ? trimmed : null; +} + +/** + * True when the payload carries an RFC 5322 threading header with content — + * i.e. this inbound email is a reply, whether or not we can resolve it to a + * ticket. Plus-addressing (`MailboxHash`) is checked separately and takes + * precedence; this is the fallback for the common case where the customer's + * mail client replies to a plain From address. + */ +export function hasReplyHeaders( + headers: Array<{ Name: string; Value: string }> | undefined, +): boolean { + return REPLY_HEADERS.some((name) => (getHeaderValue(headers, name) ?? '').trim().length > 0); +} + +/** + * Collect every candidate Message-ID a reply points at, normalized and deduped. + * + * `In-Reply-To` alone is not enough: when a customer replies to a message + * Outpost sent, it names an outbound Message-ID we never persisted. `References` + * accumulates the whole thread chain, so it still contains the customer's own + * opening Message-ID — the value stored as `Ticket.sourceId`. + * + * Order is In-Reply-To first, then the References chain as sent (oldest → + * newest). Callers match the whole set at once, so order is informational. + */ +export function extractReplyMessageIds( + headers: Array<{ Name: string; Value: string }> | undefined, +): string[] { + const ids: string[] = []; + for (const name of REPLY_HEADERS) { + const value = getHeaderValue(headers, name); + if (!value) continue; + // References is whitespace-separated; tolerate comma-separated clients. + for (const token of value.split(/[\s,]+/)) { + const id = normalizeMessageId(token); + if (id && !ids.includes(id)) ids.push(id); + } + } + return ids; +} + /** Postmark inbound webhook payload (relevant fields). */ export interface PostmarkInboundPayload { From: string; diff --git a/apps/worker/railway.toml b/apps/worker/railway.toml index d99b38bf..0dd66015 100644 --- a/apps/worker/railway.toml +++ b/apps/worker/railway.toml @@ -9,3 +9,9 @@ dockerfilePath = "apps/worker/Dockerfile" [deploy] restartPolicyType = "ALWAYS" healthcheckPath = "/health" +# A failed boot now answers 503 with its reason immediately rather than dying +# silently, so there is nothing to gain from Railway's 300s default — that +# five-minute wait per attempt is what made the 2026-08-07 incident so slow to +# read. Kept above the worker's BOOT_FAILURE_LINGER_MS (120s) so the probe sees +# the reason before the process exits to be restarted. +healthcheckTimeout = 180 diff --git a/apps/worker/src/__tests__/health.test.ts b/apps/worker/src/__tests__/health.test.ts new file mode 100644 index 00000000..770eac60 --- /dev/null +++ b/apps/worker/src/__tests__/health.test.ts @@ -0,0 +1,280 @@ +import { describe, it, expect } from 'vitest'; +import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; +import { + buildHealthResponse, + resolvePort, + summarizeBootError, + STALE_POLL_MS, + type BootState, +} from '../health.js'; + +const NOW = new Date('2026-08-12T22:00:00.000Z').getTime(); + +// Typed as the real contract, so a rename or removal in WorkerHealthStatus fails +// this file instead of leaving it green against a shape /health never serves. +const WORKER_HEALTH: WorkerHealthStatus = { + running: true, + activeJobCount: 2, + activeJobsByType: { AI_RESPONSE: 2 }, + lastPollTime: new Date(NOW - 1_000), + registeredHandlers: ['AI_RESPONSE', 'TRACKER_SYNC'], + upSince: new Date(NOW - 3_600_000), +}; + +describe('buildHealthResponse', () => { + it('reports 200 with the worker snapshot once boot is ready and the worker is polling', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); + + expect(statusCode).toBe(200); + expect(body).toMatchObject({ status: 'ok', running: true, activeJobCount: 2 }); + }); + + // `status` is the envelope's field. Spreading the snapshot over it would let a + // future WorkerHealthStatus.status redefine "ok" for every probe silently. + it('keeps its own status field even if the snapshot carries one', () => { + const shadowed = { ...WORKER_HEALTH, status: 'degraded' } as unknown as WorkerHealthStatus; + + const { body } = buildHealthResponse({ phase: 'ready' }, shadowed, NOW); + + expect(body.status).toBe('ok'); + }); + + // The regression this pins: the worker used to await the database at module + // scope, above the health server, so a boot failure exited the process before + // anything bound the port. Railway could only say "1/1 replicas never became + // healthy" — indistinguishable from a broken image, and it hid a missing + // SystemConfig table for nine days. A failed boot must now answer, and the + // answer must carry the reason. + it('reports 503 AND the reason when boot failed', () => { + const boot: BootState = { + phase: 'failed', + error: 'P2021: missing database object `public.SystemConfig`', + }; + + const { statusCode, body } = buildHealthResponse(boot, null, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('failed'); + expect(body.error).toContain('SystemConfig'); + }); + + it('reports 503 with a reason while boot is still in progress', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'starting' }, null, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('starting'); + expect(body.error).toBeTruthy(); + }); + + // Fail-fast is retained on purpose: a worker whose sync mappings could not be + // read must never be routed to, because it would write wrong statuses to + // Linear. This pins that a failed boot is not quietly downgraded to healthy. + it('never returns 200 for a failed boot, even with a worker snapshot present', () => { + const boot: BootState = { phase: 'failed', error: 'connection refused' }; + + const { statusCode, body } = buildHealthResponse(boot, WORKER_HEALTH, NOW); + + expect(statusCode).toBe(503); + expect(body.error).toBe('connection refused'); + }); + + // The gap that actually reaches production. Worker.stop() sets running=false + // without exiting the process — including from Worker's OWN signal handlers — + // so gating the 200 on the snapshot's existence alone answered + // 200 {"status":"ok","running":false} for a worker processing nothing. + it('does not report 200 for a worker that has stopped', () => { + const stopped: WorkerHealthStatus = { ...WORKER_HEALTH, running: false, upSince: null }; + + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stopped, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('stopped'); + expect(body.error).toContain('not running'); + }); + + // Worker.poll() catches every error and reschedules, so a poll blocked on a + // hung database call leaves running=true forever with lastPollTime frozen. + it('does not report 200 for a worker whose poll loop has stalled', () => { + const stalled: WorkerHealthStatus = { + ...WORKER_HEALTH, + lastPollTime: new Date(NOW - STALE_POLL_MS - 1), + }; + + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, stalled, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('stalled'); + }); + + it('does not report 200 for a worker that has never polled', () => { + const neverPolled: WorkerHealthStatus = { ...WORKER_HEALTH, lastPollTime: null }; + + const { statusCode } = buildHealthResponse({ phase: 'ready' }, neverPolled, NOW); + + expect(statusCode).toBe(503); + }); + + // A half-booted worker must not be reported healthy just because the phase + // flag says ready — and the 503 must still explain itself rather than + // answering {"status":"ready","error":null}, which is the reasonless body + // this endpoint exists to eliminate. + it('reports 503 with a reason when the phase is ready but no worker exists', () => { + const { statusCode, body } = buildHealthResponse({ phase: 'ready' }, null, NOW); + + expect(statusCode).toBe(503); + expect(body.status).toBe('no-worker'); + expect(body.error).toContain('boot sequence'); + }); + + it('serves a body that survives JSON serialization', () => { + const { body } = buildHealthResponse({ phase: 'ready' }, WORKER_HEALTH, NOW); + + expect(() => JSON.stringify(body)).not.toThrow(); + expect(JSON.parse(JSON.stringify(body))).toMatchObject({ status: 'ok', running: true }); + }); +}); + +// /health is unauthenticated, so whatever lands in boot.error is published. +// Prisma's connectivity errors quote the database host, port and user; its +// schema errors arrive as a multi-line blob whose preamble carries an absolute +// container path and a source code frame. Only the object name may survive. +describe('summarizeBootError', () => { + // The shape Prisma actually throws — not a hand-built single-line message. + const realisticP2021 = Object.assign( + new Error( + 'Invalid `prisma.systemConfig.findUnique()` invocation in\n' + + '/app/packages/outpost/shared/dist/sync/config.js:34:56\n\n' + + ' 31 const existing = await db.systemConfig.findUnique({\n\n' + + 'The table `public.SystemConfig` does not exist in the current database.', + ), + { code: 'P2021' }, + ); + + it('names the missing object without leaking container paths or the code frame', () => { + const summary = summarizeBootError(realisticP2021); + + expect(summary).toContain('P2021'); + expect(summary).toContain('public.SystemConfig'); + expect(summary).not.toContain('/app/'); + expect(summary).not.toContain('findUnique'); + }); + + it('covers P2022 missing-column drift, not just P2021', () => { + const error = Object.assign( + new Error( + 'The column `public.SystemConfig.updatedAt` does not exist in the current database.', + ), + { code: 'P2022' }, + ); + + expect(summarizeBootError(error)).toContain('P2022'); + expect(summarizeBootError(error)).toContain('SystemConfig.updatedAt'); + }); + + it('redacts the host and user out of a connectivity error', () => { + const error = Object.assign( + new Error("Can't reach database server at `db.internal.railway.app:5432`"), + { code: 'P1001' }, + ); + + const summary = summarizeBootError(error); + + expect(summary).toContain('P1001'); + expect(summary).not.toContain('db.internal.railway.app'); + expect(summary).not.toContain('5432'); + }); + + // PrismaClientInitializationError carries `errorCode`, not `code` — the real + // shape observed from a live boot against an unreachable database. + it('reads errorCode as well as code', () => { + const error = Object.assign( + new Error('Timed out fetching a new connection from the pool'), + { + errorCode: 'P2024', + }, + ); + + expect(summarizeBootError(error)).toContain('P2024'); + }); + + it('falls back to the error class when no code is present at all', () => { + const error = new Error("Can't reach database server at `127.0.0.1:59999`"); + error.name = 'PrismaClientInitializationError'; + + const summary = summarizeBootError(error); + + expect(summary).toContain('PrismaClientInitializationError'); + expect(summary).not.toContain('127.0.0.1'); + }); + + it('redacts credentials out of a plain error', () => { + const summary = summarizeBootError(new Error('postgres://user:hunter2@host/db refused')); + + expect(summary).not.toContain('hunter2'); + }); + + it('handles thrown non-Error values', () => { + expect(summarizeBootError('boom')).toBeTruthy(); + expect(summarizeBootError(null)).toBeTruthy(); + expect(summarizeBootError(undefined)).toBeTruthy(); + }); + + // A plain object carrying a safe code is the one case the echo branch exists + // for; String(obj) would render "[object Object]". + it('reads .message off a non-Error object carrying a safe code', () => { + const summary = summarizeBootError({ + code: 'P2021', + message: 'The table `public.SystemConfig` does not exist in the current database.', + }); + + expect(summary).toContain('public.SystemConfig'); + expect(summary).not.toContain('[object Object]'); + }); + + it('bounds the length of anything it serves', () => { + const error = Object.assign( + new Error(`The table \`${'x'.repeat(5_000)}\` does not exist.`), + { + code: 'P2021', + }, + ); + + expect(summarizeBootError(error).length).toBeLessThanOrEqual(200); + }); +}); + +// An invalid port used to reach server.listen() as NaN, which throws +// ERR_SOCKET_BAD_PORT synchronously at module scope — killing the process before +// anything bound, the exact opaque failure the health server exists to prevent. +describe('resolvePort', () => { + it('prefers PORT, then HEALTH_PORT, then the default', () => { + expect(resolvePort({ PORT: '8080', HEALTH_PORT: '3005' })).toMatchObject({ + port: 8080, + source: 'PORT', + }); + expect(resolvePort({ HEALTH_PORT: '3005' })).toMatchObject({ + port: 3005, + source: 'HEALTH_PORT', + }); + expect(resolvePort({})).toMatchObject({ port: 3003, source: 'default' }); + }); + + // The reported trigger: `??` only falls through on null/undefined, so a + // cleared platform variable arrives as '' and parses to NaN. + it('falls back with a warning on an empty PORT rather than yielding NaN', () => { + const resolved = resolvePort({ PORT: '', HEALTH_PORT: '3005' }); + + expect(resolved.port).toBe(3005); + expect(resolved.source).toBe('HEALTH_PORT'); + }); + + it('falls back with a warning on a non-numeric or out-of-range PORT', () => { + for (const bad of ['tcp://host:5432', 'abc', '70000', '-1']) { + const resolved = resolvePort({ PORT: bad }); + + expect(resolved.port).toBe(3003); + expect(resolved.warning).toContain(bad); + expect(Number.isInteger(resolved.port)).toBe(true); + } + }); +}); diff --git a/apps/worker/src/health.ts b/apps/worker/src/health.ts new file mode 100644 index 00000000..7834c4d4 --- /dev/null +++ b/apps/worker/src/health.ts @@ -0,0 +1,213 @@ +/** + * Boot state, health payload construction, and port resolution — split out from + * index.ts so they are testable. + * + * index.ts is a top-level-await module with side effects on import (it binds a + * port and starts polling), so its boot behaviour cannot be exercised directly + * from a test. Everything here is pure and pinned by tests: an unbooted, failed, + * stopped or stalled worker must report 503 WITH a reason, and only a worker + * that is actually polling reports 200. + */ + +import type { WorkerHealthStatus } from '@copilotkit/outpost/queue'; + +export type BootPhase = 'starting' | 'ready' | 'failed'; + +/** + * Discriminated so the failed-without-a-reason state is unrepresentable. A 503 + * carrying `error: null` is the exact signal-quality bug this module exists to + * remove; making it a type error is cheaper than remembering to assign `error` + * before `phase` on every future edit. + */ +export type BootState = + | { phase: 'starting' } + | { phase: 'ready' } + | { phase: 'failed'; error: string }; + +export interface HealthResponse { + statusCode: number; + body: Record; +} + +/** + * Prisma error codes whose failure names a schema object rather than a + * connection. P2021 is a missing table, P2022 a missing column — the drift class + * this endpoint exists to surface. Only the object name is echoed, never the + * message (see summarizeBootError). + */ +const SAFE_TO_ECHO_CODES = new Set(['P2021', 'P2022']); + +/** Worker.poll() reschedules every 1s, so a minute of silence means it is wedged. */ +export const STALE_POLL_MS = 60_000; + +/** Upper bound on any reason string served to an unauthenticated probe. */ +const MAX_REASON_LENGTH = 200; + +function messageOf(error: unknown): string { + if (error instanceof Error) return error.message; + if ( + typeof error === 'object' && + error !== null && + typeof (error as { message?: unknown }).message === 'string' + ) { + return (error as { message: string }).message; + } + return String(error); +} + +/** + * Reduce a boot exception to a reason that can be served on /health. + * + * /health is unauthenticated. Prisma's connectivity errors quote the database + * host, port and user (P1001 names host:port, P1000 names the user), and even + * the "safe" schema errors arrive as a multi-line blob whose preamble carries + * an absolute container path and a source code frame: + * + * Invalid `prisma.systemConfig.findUnique()` invocation in + * /app/packages/outpost/shared/dist/sync/config.js:34:56 + * 31 const existing = await db.systemConfig.findUnique({ + * The table `public.SystemConfig` does not exist in the current database. + * + * So nothing is echoed verbatim. For the schema codes the backticked object name + * is lifted out of the final line — that name is the entire diagnostic payload — + * and everything else degrades to error class plus code, with the full text left + * to the logs. + */ +export function summarizeBootError(error: unknown): string { + const raw = (error ?? {}) as { code?: unknown; errorCode?: unknown }; + // PrismaClientKnownRequestError carries `code`; PrismaClientInitializationError + // carries `errorCode` (frequently undefined, hence the class-name fallback). + const code = + typeof raw.code === 'string' + ? raw.code + : typeof raw.errorCode === 'string' + ? raw.errorCode + : null; + + if (code && SAFE_TO_ECHO_CODES.has(code)) { + const lastLine = messageOf(error) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .at(-1); + const object = lastLine ? /`([^`]+)`/.exec(lastLine)?.[1] : undefined; + if (object) { + return truncate( + `${code}: missing database object \`${object}\` — the database does not match schema.prisma`, + ); + } + } + + // Class name and code only. Both are stable, neither quotes the connection. + const label = [error instanceof Error ? error.name : 'Error', code].filter(Boolean).join(' '); + return truncate(`${label} — see the worker logs for the full error`); +} + +function truncate(reason: string): string { + return reason.length <= MAX_REASON_LENGTH + ? reason + : `${reason.slice(0, MAX_REASON_LENGTH - 1)}…`; +} + +/** + * Resolve the health-server port from the environment. + * + * `??` is not enough: an empty or non-numeric PORT (a cleared platform variable, + * or a reference variable that failed to resolve) parses to NaN, and + * `server.listen(NaN)` throws ERR_SOCKET_BAD_PORT synchronously at module scope + * — killing the process before anything binds, which is precisely the opaque + * "replicas never became healthy" failure this whole module exists to prevent. + * An invalid value falls back to the default and says so. + */ +export function resolvePort( + env: { PORT?: string; HEALTH_PORT?: string }, + fallback = 3003, +): { port: number; source: string; warning: string | null } { + const candidates: Array<[string, string | undefined]> = [ + ['PORT', env.PORT], + ['HEALTH_PORT', env.HEALTH_PORT], + ]; + + for (const [source, raw] of candidates) { + if (raw === undefined || raw.trim() === '') continue; + const parsed = Number.parseInt(raw, 10); + if (Number.isInteger(parsed) && parsed >= 0 && parsed <= 65535) { + return { port: parsed, source, warning: null }; + } + return { + port: fallback, + source: 'default', + warning: `invalid ${source}="${raw}" (expected an integer 0-65535), falling back to ${fallback}`, + }; + } + + return { port: fallback, source: 'default', warning: null }; +} + +/** + * Build the /health response. + * + * `workerHealth` is the worker's own snapshot, or null when the worker has not + * been constructed. It is passed rather than read so this stays pure. + * + * Every non-200 answer carries a reason. The value added over simply exiting is + * that body: a probe alone explains the failure. Exiting before binding the port + * is what made a missing SystemConfig table look identical to a broken image for + * nine days. + * + * 200 requires the worker to be *polling*, not merely constructed. Worker.stop() + * sets running=false without exiting the process, and a poll blocked on a hung + * database call freezes lastPollTime — both leave a worker that processes + * nothing while looking alive, which is the honesty gap tracked by #138. + */ +export function buildHealthResponse( + boot: BootState, + workerHealth: WorkerHealthStatus | null, + now: number = Date.now(), +): HealthResponse { + if (boot.phase === 'failed') { + return { statusCode: 503, body: { status: 'failed', error: boot.error } }; + } + + if (boot.phase === 'starting') { + return { statusCode: 503, body: { status: 'starting', error: 'boot has not finished' } }; + } + + if (!workerHealth) { + return { + statusCode: 503, + body: { + status: 'no-worker', + error: 'boot reported ready but no worker was constructed — this is a bug in the boot sequence, not a database problem', + }, + }; + } + + if (!workerHealth.running) { + return { + statusCode: 503, + body: { + ...workerHealth, + status: 'stopped', + error: 'worker is not running — it was stopped without the process exiting', + }, + }; + } + + const sincePoll = workerHealth.lastPollTime ? now - workerHealth.lastPollTime.getTime() : null; + if (sincePoll === null || sincePoll > STALE_POLL_MS) { + return { + statusCode: 503, + body: { + ...workerHealth, + status: 'stalled', + error: `worker has not polled for ${sincePoll ?? 'any'}ms — the poll loop is blocked, most likely on a hung database call`, + }, + }; + } + + // `status` last on purpose: it is the envelope's own field, and spreading the + // snapshot over it would let a future WorkerHealthStatus.status silently + // redefine what "ok" means to every probe. + return { statusCode: 200, body: { ...workerHealth, status: 'ok' } }; +} diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index 13d2b9a3..eb532317 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -15,6 +15,14 @@ * - TRACKER_SYNC: Push changes to external trackers * - JOB_CLEANUP: Periodic cleanup of old jobs and sync events * - GITHUB_REACTION_POLL: Poll GitHub reactions on AI comments (no webhook exists) + * + * BOOT ORDER: /health starts listening before any database QUERY runs, so a boot + * failure is reported rather than merely fatal. See the boot-state block below. + * Two classes still escape it, both by construction: the `prisma` import below + * constructs a PrismaClient at module scope (it throws for an ungenerated client + * or an unparseable DATABASE_URL), and a failure to bind the port itself cannot + * be reported over the port. Both are handled loudly rather than silently — see + * the health-server error handler and the last-resort handlers at the bottom. */ import http from 'node:http'; @@ -34,102 +42,303 @@ import { handleGithubReactionPoll, } from '@copilotkit/outpost/queue'; import { buildSyncEngine } from './build-sync-engine.js'; +import { buildHealthResponse, resolvePort, summarizeBootError, type BootState } from './health.js'; -// ─── Build SyncEngine for TRACKER_SYNC handler ──────────────────────────── +// ─── Boot state ─────────────────────────────────────────────────────────── -// BOOT SEMANTICS — deliberate change. This is a top-level await that performs -// three database reads (the persisted status / priority / label mapping configs) -// before this module finishes evaluating. If the database is unreachable at boot -// the import throws, so the process exits BEFORE the health server below starts -// listening: the container crash-loops with no /health at all rather than coming -// up and reporting itself degraded. +// Fail-fast on a bad boot is still the intent: a worker running with silently +// defaulted sync mappings would write wrong statuses to Linear, so it must not +// report itself healthy. What changed is that failing is no longer SILENT. // -// Fail-fast is the intent — a worker running with silently-defaulted mappings is -// worse than one that is visibly down, since TRACKER_SYNC would then write wrong -// statuses to Linear. Railway's restart policy is the retry mechanism. Note this -// interacts with the /health honesty follow-up (#138): once /health reflects -// worker state, a degraded-but-listening mode becomes a real option and this -// decision is worth revisiting. -const syncEngine = await buildSyncEngine(); - -const handleTrackerSync = createTrackerSyncHandler(syncEngine); - -// ─── Create Worker ──────────────────────────────────────────────────────── - -const worker = new Worker({ - maxConcurrency: 10, - pollIntervalMs: 1000, - concurrencyByType: { - [JobType.AI_RESPONSE]: 4, - [JobType.ESCALATION]: 2, - [JobType.SLA_CHECK]: 1, - [JobType.ONBOARDING_DIGEST]: 1, - [JobType.ACCOUNT_SCORING]: 1, - [JobType.HUBSPOT_SYNC]: 1, - [JobType.TRACKER_SYNC]: 1, - [JobType.JOB_CLEANUP]: 1, - [JobType.GITHUB_REACTION_POLL]: 1, - }, - jobTimeouts: { - [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow - [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large - [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts - }, -}); - -// ─── Register Handlers ──────────────────────────────────────────────────── - -worker.on(JobType.AI_RESPONSE, handleAiResponse); -worker.on(JobType.ESCALATION, handleEscalation); -worker.on(JobType.SLA_CHECK, handleSlaCheck); -worker.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); -worker.on(JobType.ACCOUNT_SCORING, handleAccountScoring); -worker.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); -worker.on(JobType.TRACKER_SYNC, handleTrackerSync); -worker.on(JobType.JOB_CLEANUP, handleJobCleanup); -worker.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); - -// ─── Start Scheduler ────────────────────────────────────────────────────── +// This used to be a top-level `await buildSyncEngine()` above the health server, +// so any boot-time database problem killed the process before anything bound the +// port. Railway could only report "1/1 replicas never became healthy", which is +// indistinguishable from a broken image. That cost nine days of undiagnosed +// deploy failures when SystemConfig turned out to be missing from the production +// database: every deploy from 2026-08-07 failed with no usable signal. +// +// Now the port binds first and /health answers 503 with the reason while the boot +// is unfinished or failed, so the reason is one probe away instead of buried in +// container logs nobody had reason to suspect. +// +// A failed boot does NOT park here forever. Railway's healthcheckPath gates a NEW +// DEPLOYMENT; it does not continuously probe and restart an already-running +// service, and restartPolicyType="ALWAYS" is a restart-on-exit policy that can +// never fire on a process that never exits. Staying up indefinitely would mean a +// 20-second Postgres failover during an ordinary container restart wedges the +// worker with zero jobs processed until a human notices — strictly worse than the +// crash-loop it replaced. So the reason is published for BOOT_FAILURE_LINGER_MS +// (long enough for the deploy probe and any log scrape to read it) and then the +// process exits non-zero so the restart policy retries. Diagnosable AND +// self-healing; the two were never actually in tension. +let boot: BootState = { phase: 'starting' }; -const scheduler = new Scheduler(); +let worker: Worker | null = null; +let scheduler: Scheduler | null = null; // ─── Health Server ──────────────────────────────────────────────────────── -const port = parseInt(process.env.PORT ?? process.env.HEALTH_PORT ?? '3003', 10); +const { port, source: portSource, warning: portWarning } = resolvePort(process.env); +if (portWarning) console.error(`[Worker] ${portWarning}`); const healthServer = http.createServer((req, res) => { - if (req.url === '/health') { - const health = worker.healthCheck(); - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok', ...health })); - } else { - res.writeHead(404); - res.end('Not Found'); + // A probe must never be able to kill the process it exists to observe: + // worker.healthCheck() and JSON.stringify both run in this callback, and an + // exception in an http listener is an uncaught exception. + try { + const path = + new URL(req.url ?? '/', 'http://localhost').pathname.replace(/\/+$/, '') || '/'; + if (path !== '/health') { + res.writeHead(404, { 'Content-Type': 'text/plain' }); + res.end('Not Found'); + return; + } + + const { statusCode, body } = buildHealthResponse( + boot, + worker ? worker.healthCheck() : null, + ); + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); + } catch (error) { + console.error('[Worker] /health handler threw:', error); + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + status: 'error', + error: 'health handler failed — see the worker logs', + }), + ); } }); -// ─── Start Everything ───────────────────────────────────────────────────── +// listen() reports bind failures asynchronously through 'error'. With no listener +// that is an uncaught exception: the port never binds and the process dies to a +// bare stack trace — the same opaque signal as the original incident, arriving +// through the one step everything else now depends on. It is also the single +// failure that genuinely cannot be reported over /health, so it must be loud in +// the logs and must exit rather than linger pretending to serve. +healthServer.on('error', (error: NodeJS.ErrnoException) => { + console.error( + `[Worker] FATAL: could not bind the health server to port ${port} (${error.code ?? 'unknown'}, from ${portSource}). ` + + `Nothing can report this process's state without it. Check PORT/HEALTH_PORT and whether another process holds the port.`, + error, + ); + process.exit(1); +}); healthServer.listen(port, () => { - console.log(`[Worker] Health server listening on port ${port}`); + console.log( + `[Worker] Health server listening on port ${port} (from ${portSource}, boot: ${boot.phase})`, + ); }); -scheduler.start(); -worker.start(); +// ─── Graceful Shutdown ──────────────────────────────────────────────────── -console.log('[Worker] Worker process started'); +// Registered BEFORE the boot await, not after it. Boot is the slowest thing this +// process does, which is exactly when Railway tears a bad deploy down — and a +// SIGTERM arriving while module evaluation is still suspended would find no +// handler and kill the process outright. +// +// Note there is a SECOND registrar: Worker.start() installs its own SIGTERM / +// SIGINT handlers that call worker.stop() unawaited. Both fire. That is safe +// only because Worker.stop() early-returns on !running and this handler runs +// first, so ordering here is load-bearing — do not move this registration below +// startWorker(). +let shuttingDown = false; -// ─── Graceful Shutdown ──────────────────────────────────────────────────── +// Longer than the largest entry in jobTimeouts below (300s), because +// Worker.stop() waits for in-flight jobs to finish. A watchdog shorter than the +// drain it guards would turn every deploy that lands mid-job into a forced +// exit(1) — guarding the hang while breaking the normal path. +const SHUTDOWN_WATCHDOG_MS = Number(process.env.SHUTDOWN_WATCHDOG_MS ?? 330_000); async function shutdown(signal: string): Promise { - console.log(`[Worker] Received ${signal}, shutting down...`); - scheduler.stop(); - await worker.stop(); - healthServer.close(); - await prisma.$disconnect(); - console.log('[Worker] Shutdown complete'); - process.exit(0); + if (shuttingDown) return; + shuttingDown = true; + console.log(`[Worker] Received ${signal} in boot phase '${boot.phase}', shutting down...`); + + // NOT unref'd. The motivating case is a $disconnect() that never settles + // after the server is closed — precisely when no other referenced handle + // remains, so an unref'd timer would let Node exit 0 (reporting a clean stop + // for a shutdown that never completed) and this line would never print. + // Every path below ends in process.exit, so a referenced timer costs nothing. + const watchdog = setTimeout(() => { + console.error( + `[Worker] Shutdown did not finish in ${SHUTDOWN_WATCHDOG_MS}ms, exiting anyway`, + ); + process.exit(1); + }, SHUTDOWN_WATCHDOG_MS); + + try { + // Close the listener FIRST. Worker.stop() blocks until in-flight jobs + // finish (up to 300s), and advertising a healthy /health for the whole + // drain window tells the platform to keep routing to a replica that has + // already committed to dying. + healthServer.close(); + scheduler?.stop(); + await worker?.stop(); + await prisma.$disconnect(); + console.log('[Worker] Shutdown complete'); + process.exit(0); + } catch (error) { + // Observed: signalled mid-boot, $disconnect() rejects while tearing down + // a pool that never filled ("Timed out fetching a new connection from the + // connection pool"). Without this the rejection is unhandled and the + // process dies to a stack trace mid-shutdown instead of reporting a + // failed stop. + console.error('[Worker] Shutdown failed:', error); + process.exit(1); + } +} + +// `void` because an unhandled rejection here would be the very failure the catch +// above exists to prevent. +process.on('SIGTERM', () => void shutdown('SIGTERM')); +process.on('SIGINT', () => void shutdown('SIGINT')); + +// ─── Last-Resort Handlers ───────────────────────────────────────────────── + +// The whole design rests on this process staying up to explain itself, and under +// Node's defaults a single unhandled rejection ends it with a bare stack trace — +// back to the undiagnosable behaviour. Worker.poll() is fired unawaited from a +// timer and Worker's own signal handler calls stop() unawaited, so the paths +// exist. Mark the process unhealthy so /health tells the platform to stop routing +// to it, publish the reason, and then exit so the restart policy retries rather +// than leaving a wedged replica behind. +function failFatally(kind: string, error: unknown): void { + console.error(`[Worker] ${kind}:`, error); + if (boot.phase !== 'failed') { + boot = { phase: 'failed', error: `${kind}: ${summarizeBootError(error)}` }; + } + if (!shuttingDown) { + setTimeout(() => process.exit(1), 5_000); + } +} + +process.on('unhandledRejection', (reason) => failFatally('UNHANDLED REJECTION', reason)); +process.on('uncaughtException', (error) => failFatally('UNCAUGHT EXCEPTION', error)); + +// ─── Boot ───────────────────────────────────────────────────────────────── + +// Everything that can throw at boot lives in here: buildSyncEngine's three +// database reads (the persisted status / priority / label mapping configs), the +// Worker construction, and the scheduler/worker start. Anything that escapes +// leaves boot.phase === 'failed' and the process ALIVE but unhealthy, so the +// reason reaches /health instead of vanishing with the process. +async function startWorker(): Promise { + const syncEngine = await buildSyncEngine(); + const handleTrackerSync = createTrackerSyncHandler(syncEngine); + + const started = new Worker({ + maxConcurrency: 10, + pollIntervalMs: 1000, + concurrencyByType: { + [JobType.AI_RESPONSE]: 4, + [JobType.ESCALATION]: 2, + [JobType.SLA_CHECK]: 1, + [JobType.ONBOARDING_DIGEST]: 1, + [JobType.ACCOUNT_SCORING]: 1, + [JobType.HUBSPOT_SYNC]: 1, + [JobType.TRACKER_SYNC]: 1, + [JobType.JOB_CLEANUP]: 1, + [JobType.GITHUB_REACTION_POLL]: 1, + }, + jobTimeouts: { + [JobType.AI_RESPONSE]: 120_000, // 2 minutes — AI pipeline is slow + [JobType.HUBSPOT_SYNC]: 300_000, // 5 minutes — full sync can be large + [JobType.ACCOUNT_SCORING]: 300_000, // 5 minutes — many accounts + }, + }); + + // ─── Register Handlers ──────────────────────────────────────────────── + started.on(JobType.AI_RESPONSE, handleAiResponse); + started.on(JobType.ESCALATION, handleEscalation); + started.on(JobType.SLA_CHECK, handleSlaCheck); + started.on(JobType.ONBOARDING_DIGEST, handleOnboardingDigest); + started.on(JobType.ACCOUNT_SCORING, handleAccountScoring); + started.on(JobType.HUBSPOT_SYNC, handleHubSpotSync); + started.on(JobType.TRACKER_SYNC, handleTrackerSync); + started.on(JobType.JOB_CLEANUP, handleJobCleanup); + started.on(JobType.GITHUB_REACTION_POLL, handleGithubReactionPoll); + + // A SIGTERM can land while buildSyncEngine() is still awaiting. shutdown() + // then runs to completion against null handles and heads for process.exit, + // and without this check the boot would resume behind it: Scheduler.start() + // ticks every definition immediately (enqueueing jobs) and Worker.start() + // begins claiming them, so the exit strands freshly-claimed rows in + // PROCESSING. Nothing sequences the two promise chains, so the flag is what + // sequences them. + if (shuttingDown) { + console.log('[Worker] Boot completed after shutdown began — not starting the worker'); + return; + } + + // Published before start() so a probe landing mid-start sees the real worker, + // and so shutdown can stop it if a signal arrives during boot. + const nextScheduler = new Scheduler(); + worker = started; + scheduler = nextScheduler; + + try { + nextScheduler.start(); + started.start(); + } catch (error) { + // Scheduler.start() ticks every definition immediately and installs + // setInterval timers, so a throw between it and worker.start() would + // otherwise leave a process that reports itself failed while still + // enqueueing jobs nothing will consume. Torn down through the locals — + // the module-level handles are narrowed to null at the outer catch. + nextScheduler.stop(); + await started.stop().catch(() => {}); + worker = null; + scheduler = null; + throw error; + } } -process.on('SIGTERM', () => shutdown('SIGTERM')); -process.on('SIGINT', () => shutdown('SIGINT')); +// NOTHING IS RETHROWN HERE, deliberately. This is a top-level-await entry +// module: an exception escaping module evaluation rejects its evaluation +// promise, which Node reports as an uncaught exception and exits on — a +// listening HTTP server does not keep the process alive. Rethrowing would kill +// the health server before it could answer a single probe and hand Railway the +// same bare "1/1 replicas never became healthy" that hid a missing SystemConfig +// table for nine days. Staying up and answering 503 IS the fix. +// +// Fail-fast is still the intent: a worker whose sync mappings could not be read +// must never be reported healthy, because TRACKER_SYNC would write wrong +// statuses to Linear. Railway fails the deploy on the failing healthcheck and +// keeps the previous replica serving — same outcome, with a reason attached. +// How long a failed boot keeps answering 503 with its reason before exiting so +// restartPolicyType="ALWAYS" retries. Long enough for a deploy healthcheck and a +// log scrape to read it; short enough that a transient database outage recovers +// on its own rather than waiting for a human. +const BOOT_FAILURE_LINGER_MS = Number(process.env.BOOT_FAILURE_LINGER_MS ?? 120_000); + +try { + await startWorker(); + if (!shuttingDown) { + boot = { phase: 'ready' }; + console.log('[Worker] Worker process started'); + } +} catch (error) { + // startWorker() has already torn down anything it managed to start, so by + // here the process holds no timers and no poll loop — only the health server. + // + // The full error goes to the logs only — /health carries the redacted form, + // since Prisma's errors quote the database host, port, user and container paths. + boot = { phase: 'failed', error: summarizeBootError(error) }; + console.error('[Worker] BOOT FAILED:', error); + console.error( + `[Worker] /health on ${port} reports 503 ("${boot.error}") for ${BOOT_FAILURE_LINGER_MS}ms, ` + + `then this process exits 1 so Railway's restart policy retries. ` + + `A missing table or column here means the database does not match schema.prisma — ` + + `check the schema-drift guard in apps/worker/start.sh.`, + ); + setTimeout(() => { + console.error( + '[Worker] Exiting after the boot-failure linger window; restart policy takes over.', + ); + process.exit(1); + }, BOOT_FAILURE_LINGER_MS); +} diff --git a/packages/outpost/ai/src/generator.test.ts b/packages/outpost/ai/src/generator.test.ts index feb43a96..659f319c 100644 --- a/packages/outpost/ai/src/generator.test.ts +++ b/packages/outpost/ai/src/generator.test.ts @@ -5,6 +5,7 @@ import { SYSTEM_PROMPT_PREFIX, ResponseGenerator, buildChannelGuidance, + extractResponseText, } from './generator.js'; import { ConfidenceLevel } from './types.js'; import type { SearchResult } from './types.js'; @@ -61,6 +62,30 @@ describe('ResponseGenerator', () => { }); describe('generate', () => { + it('should extract text across response blocks when the first block is non-text', () => { + const content = [ + { type: 'tool_use', id: 'tool-1', name: 'lookup', input: {} }, + { type: 'text', text: 'First part' }, + { type: 'text', text: ' and second part' }, + ] as unknown as Parameters[0]; + + expect(extractResponseText(content)).toBe('First part and second part'); + expect(extractResponseText([])).toBe(''); + }); + + it('should return the safe fallback when the model produces no usable text', async () => { + mock.onMessage(/./, { + content: '', + usage: { input_tokens: 100, output_tokens: 0 }, + }); + + const result = await generator.generate({ question: 'test' }, sampleSources); + + expect(result.text).toContain('unable to generate'); + expect(result.confidenceScore).toBe(0); + expect(result.degraded).toBe(true); + }); + it('should generate a response with confidence scoring', async () => { mock.onMessage(/./, { content: 'Here is how to use CopilotKit actions...', diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index c38263b3..0bcde3c4 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -80,6 +80,11 @@ export function buildChannelGuidance(source?: PlatformTarget): string { ].join('\n'); } +/** Extract all text blocks from an Anthropic response in response order. */ +export function extractResponseText(content: Anthropic.ContentBlock[]): string { + return content.map((block) => (block.type === 'text' ? block.text : '')).join(''); +} + /** * Claude response generator for the AI support pipeline. * @@ -119,8 +124,10 @@ export class ResponseGenerator { messages, }); - const responseText = - message.content[0].type === 'text' ? message.content[0].text : ''; + const responseText = extractResponseText(message.content); + if (!responseText.trim()) { + throw new Error('Model response contained no usable text'); + } const tokenUsage: TokenUsage = { inputTokens: message.usage.input_tokens, diff --git a/packages/outpost/db/prisma/migrations/20260811193000_add_message_response_key/migration.sql b/packages/outpost/db/prisma/migrations/20260811193000_add_message_response_key/migration.sql new file mode 100644 index 00000000..0f7496f0 --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260811193000_add_message_response_key/migration.sql @@ -0,0 +1,7 @@ +-- Add a nullable idempotency slot rather than constraining historical BOT +-- messages. PostgreSQL permits multiple NULLs in a unique index, so existing +-- rows remain valid while new primary AI responses claim one slot per ticket. +ALTER TABLE "Message" ADD COLUMN "responseKey" TEXT; + +CREATE UNIQUE INDEX "Message_ticketId_responseKey_key" +ON "Message"("ticketId", "responseKey"); diff --git a/packages/outpost/db/prisma/migrations/20260811200000_add_message_response_state/migration.sql b/packages/outpost/db/prisma/migrations/20260811200000_add_message_response_state/migration.sql new file mode 100644 index 00000000..c42ba73a --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260811200000_add_message_response_state/migration.sql @@ -0,0 +1,6 @@ +CREATE TYPE "MessageResponseState" AS ENUM ('PENDING', 'DELIVERED', 'ESCALATED'); + +ALTER TABLE "Message" +ADD COLUMN "responseState" "MessageResponseState", +ADD COLUMN "responseJobId" TEXT, +ADD COLUMN "responseError" TEXT; diff --git a/packages/outpost/db/prisma/migrations/20260811220000_add_job_claim_token/migration.sql b/packages/outpost/db/prisma/migrations/20260811220000_add_job_claim_token/migration.sql new file mode 100644 index 00000000..48992577 --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260811220000_add_job_claim_token/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "Job" +ADD COLUMN "claimToken" TEXT; diff --git a/packages/outpost/db/prisma/migrations/20260811230000_add_unique_postmark_message_id/migration.sql b/packages/outpost/db/prisma/migrations/20260811230000_add_unique_postmark_message_id/migration.sql new file mode 100644 index 00000000..4d42bd90 --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260811230000_add_unique_postmark_message_id/migration.sql @@ -0,0 +1,6 @@ +-- Postmark retries carry the same MessageID. Enforce one EMAIL ticket per +-- delivery at the database boundary while leaving other platform source-key +-- policies unchanged. +CREATE UNIQUE INDEX "Ticket_email_sourceId_key" +ON "Ticket"("sourceId") +WHERE "source" = 'EMAIL' AND "sourceId" IS NOT NULL; diff --git a/packages/outpost/db/prisma/schema.prisma b/packages/outpost/db/prisma/schema.prisma index cb06fb03..69ee89be 100644 --- a/packages/outpost/db/prisma/schema.prisma +++ b/packages/outpost/db/prisma/schema.prisma @@ -108,9 +108,14 @@ model Message { confidenceLevel String? // "HIGH" | "MEDIUM" | "LOW" feedback String? // "POSITIVE" | "NEGATIVE" | null (no feedback yet) externalCommentId String? // GitHub comment/discussion-comment ID this message was posted as, if applicable + responseKey String? // Idempotency slot; PRIMARY_AI_RESPONSE is unique per ticket + responseState MessageResponseState? // Lifecycle of the primary AI response + responseJobId String? // Queue job that owns a PENDING primary response + responseError String? // Last delivery failure, used by retry recovery attachments Json? createdAt DateTime @default(now()) + @@unique([ticketId, responseKey]) @@index([ticketId]) @@index([isAiGenerated, feedback]) } @@ -121,6 +126,12 @@ enum MessageType { SYSTEM } +enum MessageResponseState { + PENDING + DELIVERED + ESCALATED +} + model Note { id String @id @default(cuid()) ticketId String @@ -205,6 +216,7 @@ model Job { progress Int? // Optional percentage 0-100 runAt DateTime @default(now()) lockedAt DateTime? + claimToken String? // Fences writes to the worker execution that owns this claim completedAt DateTime? error String? createdAt DateTime @default(now()) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index a717ff6b..89f16ede 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -18,16 +18,22 @@ const mockPrismaTicket = { const mockPrismaMessage = { create: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), }; const mockPrismaJob = { create: vi.fn(), }; +const mockPrismaQueryRaw = vi.fn(); +const mockPrismaTransaction = vi.fn(); + const mockPrisma = { ticket: mockPrismaTicket, message: mockPrismaMessage, job: mockPrismaJob, + $queryRaw: mockPrismaQueryRaw, + $transaction: mockPrismaTransaction, }; vi.mock('@copilotkit/outpost/db', () => ({ @@ -89,10 +95,13 @@ const { handleAiResponse } = await import('../handlers/ai-response.js'); // ─── Test Helpers ────────────────────────────────────────────────────────── -function makeContext(): JobHandlerContext { +const PENDING_RECOVERY_AFTER_MS = 5 * 60 * 1000; + +function makeContext(overrides: Partial = {}): JobHandlerContext { return { jobId: 'test-job-1', reportProgress: vi.fn().mockResolvedValue(undefined), + ...overrides, }; } @@ -239,7 +248,12 @@ describe('handleAiResponse', () => { mockPrismaTicket.update.mockResolvedValue({}); mockPrismaMessage.create.mockResolvedValue({ id: 'msg-new' }); mockPrismaMessage.update.mockResolvedValue({}); + mockPrismaMessage.updateMany.mockResolvedValue({ count: 1 }); mockPrismaJob.create.mockResolvedValue({ id: 'job-esc-1' }); + mockPrismaQueryRaw.mockResolvedValue([{ now: new Date('2026-08-11T20:00:00.000Z') }]); + mockPrismaTransaction.mockImplementation( + async (callback: (tx: typeof mockPrisma) => Promise) => callback(mockPrisma), + ); mockGenerateSupportResponse.mockResolvedValue(highConfidenceResult); mockClassifyTicket.mockResolvedValue(sampleClassification); mockGetFeedbackCalibration.mockResolvedValue(0); @@ -301,12 +315,7 @@ describe('handleAiResponse', () => { 'How do I use CopilotKit with Next.js?', expect.objectContaining({ source: 'discord', - conversationHistory: expect.arrayContaining([ - expect.objectContaining({ - role: 'user', - content: 'How do I use CopilotKit with Next.js?', - }), - ]), + conversationHistory: [], }), ); }); @@ -518,6 +527,10 @@ describe('handleAiResponse', () => { isAiGenerated: true, confidenceScore: 0.92, confidenceLevel: 'HIGH', + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'test-job-1', + responseError: null, }, }); }); @@ -859,7 +872,8 @@ describe('handleAiResponse', () => { ); // The failed write must not abort the job between the BOT row and the - // post-back — that is the window the guard makes unrecoverable. + // post-back. Recovery is intentionally human-only, so this attempt + // should still use its one chance to deliver automatically. expect(mockPostResponse).toHaveBeenCalled(); expect(result.success).toBe(true); expect(result.data?.deliveryFailed).toBe(false); @@ -898,14 +912,79 @@ describe('handleAiResponse', () => { expect(mockPrismaJob.create).not.toHaveBeenCalled(); }); + it('does not escalate when posting succeeds but DELIVERED state persistence fails', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockPrismaMessage.update.mockImplementation( + async (args: { data: Record }) => { + if (args.data.responseState === 'DELIVERED') { + throw new Error('DB write conflict'); + } + return {}; + }, + ); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-delivered-state' }), + ); + + expect(mockPostResponse).toHaveBeenCalledTimes(1); + expect(result.success).toBe(true); + expect(result.data?.deliveryFailed).toBe(false); + expect(result.data?.escalated).toBe(false); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(mockPrismaMessage.update).toHaveBeenCalledWith({ + where: { id: 'msg-new' }, + data: { + responseError: expect.stringContaining('DELIVERY_CONFIRMED'), + }, + }); + }); + + it('does not escalate or repost a confirmed delivery whose DELIVERED state write failed', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-delivered-state-failed', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-delivered-state', + responseError: 'DELIVERY_CONFIRMED: DB write conflict', + createdAt: new Date(), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-delivered-state' }), + ); + + expect(result.success).toBe(true); + expect(result.data).toMatchObject({ skipped: true, reason: 'already_answered' }); + expect(mockPostResponse).not.toHaveBeenCalled(); + expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(mockPrismaMessage.update).toHaveBeenCalledWith({ + where: { id: 'msg-delivered-state-failed' }, + data: { responseState: 'DELIVERED', responseError: null }, + }); + }); + it('reports job failure when the answer was neither delivered nor escalated', async () => { mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + const context = makeContext(); const result = await handleAiResponse( { ticketId: 'tkt-1', source: 'discord' }, - makeContext(), + context, ); // Nothing reached the reporter and no human was pulled in; a silent @@ -913,22 +992,293 @@ describe('handleAiResponse', () => { expect(result.success).toBe(false); expect(result.error).toContain('Discord API 503'); expect(result.error).toContain('queue unavailable'); + // A terminal worker attempt preserves the handler's last progress + // value on the DEAD_LETTER row. Failure must therefore stop at the + // last completed phase instead of looking 100% complete. + expect(context.reportProgress).not.toHaveBeenCalledWith(100); + expect(context.reportProgress).toHaveBeenLastCalledWith(85); }); - it('still reports success when only a low-confidence escalation fails to enqueue', async () => { - // The reporter did get the answer here, so the historical fail-soft - // behaviour stands — the failure mode is different in kind. - mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + it('retries the escalation after delivery and escalation both fail', async () => { + const pendingResponse = { + id: 'msg-new', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-retry-delivery', + responseError: 'Discord API 503', + createdAt: new Date(Date.now() - PENDING_RECOVERY_AFTER_MS - 1), + }; + mockPrismaTicket.findUnique + .mockResolvedValueOnce(sampleTicket) + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [...sampleTicket.messages, pendingResponse], + }) + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { ...pendingResponse, responseJobId: 'job-delayed-delivery-recovery' }, + ], + }); + mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); + mockPrismaJob.create + .mockRejectedValueOnce(new Error('queue unavailable')) + .mockResolvedValueOnce({ id: 'job-delayed-delivery-recovery' }) + .mockResolvedValueOnce({ id: 'job-escalation-retry' }); + + const context = makeContext({ jobId: 'job-retry-delivery' }); + const firstAttempt = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + const retryAttempt = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + const recoveryAttempt = await handleAiResponse( + { + ticketId: 'tkt-1', + source: 'discord', + pendingResponseRecovery: { messageId: pendingResponse.id }, + }, + makeContext({ jobId: 'job-delayed-delivery-recovery' }), + ); + + expect(firstAttempt.success).toBe(false); + expect(retryAttempt.success).toBe(true); + expect(retryAttempt.data).toMatchObject({ recoveryScheduled: true }); + expect(recoveryAttempt.data).toMatchObject({ + skipped: true, + escalated: true, + reason: 'delivery_recovered', + }); + expect(mockPrismaJob.create).toHaveBeenCalledTimes(3); + expect(mockPostResponse).toHaveBeenCalledTimes(1); + expect(mockGenerateSupportResponse).toHaveBeenCalledTimes(1); + }); + + it('escalates on retry after interruption between delivery failure and escalation', async () => { + const pendingResponse = { + id: 'msg-new', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-interrupted', + responseError: 'Discord API 503', + createdAt: new Date(Date.now() - PENDING_RECOVERY_AFTER_MS - 1), + }; + mockPrismaTicket.findUnique + .mockResolvedValueOnce(sampleTicket) + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [...sampleTicket.messages, pendingResponse], + }) + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { ...pendingResponse, responseJobId: 'job-delayed-interruption' }, + ], + }); + mockPostResponse.mockRejectedValueOnce(new Error('Discord API 503')); + mockPrismaJob.create + .mockResolvedValueOnce({ id: 'job-delayed-interruption' }) + .mockResolvedValueOnce({ id: 'job-escalation-after-interruption' }); + + let interruptAt85 = true; + const firstProgress = vi.fn().mockImplementation(async (percent: number) => { + if (percent === 85 && interruptAt85) { + interruptAt85 = false; + throw new Error('worker interrupted after response row was created'); + } + }); + await expect( + handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-interrupted', reportProgress: firstProgress }), + ), + ).rejects.toThrow('worker interrupted'); + + const retryAttempt = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-interrupted' }), + ); + const recoveryAttempt = await handleAiResponse( + { + ticketId: 'tkt-1', + source: 'discord', + pendingResponseRecovery: { messageId: pendingResponse.id }, + }, + makeContext({ jobId: 'job-delayed-interruption' }), + ); + + expect(retryAttempt.success).toBe(true); + expect(retryAttempt.data).toMatchObject({ recoveryScheduled: true }); + expect(recoveryAttempt.data).toMatchObject({ + skipped: true, + escalated: true, + reason: 'delivery_recovered', + }); + expect(mockPrismaJob.create).toHaveBeenCalledTimes(2); + expect(mockPostResponse).toHaveBeenCalledTimes(1); + expect(mockGenerateSupportResponse).toHaveBeenCalledTimes(1); + }); + + it.each([ + ['low-confidence', lowConfidenceResult, 'Low AI confidence'], + ['suppressed', suppressedResult, 'AI response withheld'], + ])( + 'fails durably when a delivered %s response cannot enqueue its required escalation', + async (_label, pipelineResult, reasonFragment) => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(pipelineResult); + mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-required-escalation' }), + ); + + expect(mockPostResponse).toHaveBeenCalledTimes(1); + expect(result.success).toBe(false); + expect(result.error).toContain('queue unavailable'); + expect(mockPrismaMessage.update).toHaveBeenCalledWith({ + where: { id: 'msg-new' }, + data: { + responseError: expect.stringContaining( + `ESCALATION_REQUIRED: ${reasonFragment}`, + ), + }, + }); + expect(mockPrismaMessage.update).not.toHaveBeenCalledWith({ + where: { id: 'msg-new' }, + data: { responseState: 'DELIVERED', responseError: null }, + }); + }, + ); + + it('retries a required escalation without regenerating or reposting the delivered response', async () => { + const pendingRequiredEscalation = { + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-required-escalation', + type: 'BOT', + content: lowConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-required-escalation', + responseError: + 'ESCALATION_REQUIRED: Low AI confidence (25%) — automated escalation', + createdAt: new Date(), + }, + ], + }; + mockPrismaTicket.findUnique + .mockResolvedValueOnce(sampleTicket) + .mockResolvedValueOnce(pendingRequiredEscalation); mockGenerateSupportResponse.mockResolvedValue(lowConfidenceResult); - mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + mockPrismaJob.create + .mockRejectedValueOnce(new Error('queue unavailable')) + .mockResolvedValueOnce({ id: 'job-required-escalation-retry' }); + + const context = makeContext({ jobId: 'job-required-escalation' }); + const firstAttempt = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + const retryAttempt = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + + expect(firstAttempt.success).toBe(false); + expect(retryAttempt.success).toBe(true); + expect(retryAttempt.data).toMatchObject({ + skipped: true, + escalated: true, + reason: 'escalation_recovered', + }); + expect(mockGenerateSupportResponse).toHaveBeenCalledTimes(1); + expect(mockPostResponse).toHaveBeenCalledTimes(1); + + const escalationCalls = mockPrismaJob.create.mock.calls.filter( + (call: Array<{ data: { type: string } }>) => call[0].data.type === 'ESCALATION', + ); + // One failed enqueue plus one successful retry; no additional job is + // created once the ordinary retry completes. + expect(escalationCalls).toHaveLength(2); + expect(escalationCalls[1][0].data.payload).toEqual({ + ticketId: 'tkt-1', + reason: 'Low AI confidence (25%) — automated escalation', + }); + expect(mockPrismaMessage.updateMany).toHaveBeenCalledWith({ + where: { + id: 'msg-required-escalation', + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + }, + data: { responseState: 'ESCALATED', responseError: null }, + }); + }); + + it('recovers the keyed primary response when an older AI BOT row appears first', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'legacy-ai-row', + type: 'BOT', + content: 'Legacy AI response', + isAiGenerated: true, + responseKey: null, + responseState: null, + createdAt: new Date('2026-04-23T10:00:10Z'), + }, + { + id: 'primary-ai-row', + type: 'BOT', + content: lowConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-required-escalation', + responseError: + 'ESCALATION_REQUIRED: Low AI confidence (25%) — automated escalation', + createdAt: new Date('2026-04-23T10:00:20Z'), + }, + ], + }); const result = await handleAiResponse( { ticketId: 'tkt-1', source: 'discord' }, - makeContext(), + makeContext({ jobId: 'job-required-escalation' }), ); - expect(mockPostResponse).toHaveBeenCalled(); expect(result.success).toBe(true); + expect(result.data).toMatchObject({ reason: 'escalation_recovered' }); + expect(mockPrismaJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ type: 'ESCALATION' }), + }); + expect(mockPrismaMessage.updateMany).toHaveBeenCalledWith({ + where: { + id: 'primary-ai-row', + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + }, + data: { responseState: 'ESCALATED', responseError: null }, + }); + expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); }); }); @@ -1046,7 +1396,6 @@ describe('handleAiResponse', () => { 'Hello', expect.objectContaining({ conversationHistory: [ - { role: 'user', content: 'Hello' }, { role: 'assistant', content: 'Hi there!' }, { role: 'user', content: 'Follow up question' }, ], @@ -1098,17 +1447,17 @@ describe('handleAiResponse', () => { ); }); - it('still passes the interim follow-up through as conversation context', async () => { + it('passes the follow-up as context without repeating the opening question', async () => { mockPrismaTicket.findUnique.mockResolvedValue(splitThoughtTicket); await handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()); - expect(mockGenerateSupportResponse.mock.calls[0]?.[1]).toMatchObject({ - conversationHistory: [ - { role: 'user', content: 'How do I use CopilotKit with Next.js?' }, - { role: 'user', content: 'btw I am on the app router' }, - ], - }); + expect(mockGenerateSupportResponse).toHaveBeenCalledWith( + 'How do I use CopilotKit with Next.js?', + expect.objectContaining({ + conversationHistory: [{ role: 'user', content: 'btw I am on the app router' }], + }), + ); }); it('skips leading non-USER rows to find the opening USER message', async () => { @@ -1215,6 +1564,270 @@ describe('handleAiResponse', () => { expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); }); + it('lets only one of two concurrent handlers post a response', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + + // Hold both model calls until both handlers have passed the initial + // history check. This makes the check-then-insert race deterministic: + // neither invocation can observe the other's response in its ticket + // snapshot. + let generatorsStarted = 0; + let releaseGenerators!: () => void; + const bothGeneratorsStarted = new Promise((resolve) => { + releaseGenerators = resolve; + }); + mockGenerateSupportResponse.mockImplementation(async () => { + generatorsStarted += 1; + if (generatorsStarted === 2) releaseGenerators(); + await bothGeneratorsStarted; + return highConfidenceResult; + }); + + // Model the database's unique (ticketId, responseKey) constraint. + // Before the production insert supplies responseKey, both writes + // succeed and this test fails with two platform posts. + let responseClaimed = false; + mockPrismaMessage.create.mockImplementation( + async (args: { data: { responseKey?: string } }) => { + if (args.data.responseKey === 'PRIMARY_AI_RESPONSE') { + if (responseClaimed) { + throw { + code: 'P2002', + meta: { target: ['ticketId', 'responseKey'] }, + }; + } + responseClaimed = true; + } + return { id: `msg-${responseClaimed ? 'winner' : 'unclaimed'}` }; + }, + ); + + const results = await Promise.all([ + handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()), + handleAiResponse({ ticketId: 'tkt-1', source: 'discord' }, makeContext()), + ]); + + expect(generatorsStarted).toBe(2); + expect(mockPostResponse).toHaveBeenCalledTimes(1); + expect(results.filter((result) => result.data?.skipped)).toHaveLength(1); + expect(results.every((result) => result.success)).toBe(true); + }); + + it('does not recover a fresh PENDING response owned by another job', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-in-flight', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-still-posting', + createdAt: new Date(), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-concurrent-loser' }), + ); + + expect(result.data).toMatchObject({ skipped: true, reason: 'already_answered' }); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + + it('schedules exactly one delayed takeover for a fresh same-job PENDING response', async () => { + const now = Date.parse('2026-08-11T20:00:00.000Z'); + const dateNow = vi.spyOn(Date, 'now').mockReturnValue(now); + const pendingResponse = { + id: 'msg-same-job-in-flight', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-timed-out', + createdAt: new Date(now - 1_000), + }; + mockPrismaTicket.findUnique + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [...sampleTicket.messages, pendingResponse], + }) + .mockResolvedValueOnce({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { ...pendingResponse, responseJobId: 'job-delayed-takeover' }, + ], + }); + mockPrismaJob.create.mockResolvedValueOnce({ id: 'job-delayed-takeover' }); + + try { + const firstRetry = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-timed-out' }), + ); + const duplicateRetry = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-timed-out' }), + ); + + expect(firstRetry.data).toMatchObject({ + skipped: true, + recoveryScheduled: true, + reason: 'delivery_recovery_scheduled', + }); + expect(duplicateRetry.data).toMatchObject({ + skipped: true, + reason: 'already_answered', + }); + expect(mockPrismaJob.create).toHaveBeenCalledTimes(1); + expect(mockPrismaJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + type: 'AI_RESPONSE', + payload: { + ticketId: 'tkt-1', + source: 'discord', + pendingResponseRecovery: { + messageId: 'msg-same-job-in-flight', + }, + }, + // The due time is based on the database clock fixture, + // not message.createdAt or Date.now(). + runAt: new Date('2026-08-11T20:05:00.000Z'), + }), + }); + expect(mockPrismaMessage.updateMany).toHaveBeenCalledWith({ + where: { + id: 'msg-same-job-in-flight', + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-timed-out', + }, + data: { responseJobId: 'job-delayed-takeover' }, + }); + expect(mockPostResponse).not.toHaveBeenCalled(); + } finally { + dateNow.mockRestore(); + } + }); + + it('skips a delayed takeover when the original response became DELIVERED', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-delivered-before-takeover', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'DELIVERED', + responseJobId: 'job-delayed-takeover', + createdAt: new Date(Date.now() - PENDING_RECOVERY_AFTER_MS), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-delayed-takeover' }), + ); + + expect(result.data).toMatchObject({ skipped: true, reason: 'already_answered' }); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + + it('recovers only from an explicit delayed payload even when the app clock is behind', async () => { + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-stale', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-takeover', + // Deliberately in the app clock's future. Recovery must + // be authorized by the durable payload/owner pair, not + // by Date.now() arithmetic against a DB timestamp. + createdAt: new Date('2099-01-01T00:00:00.000Z'), + }, + ], + }); + + const result = await handleAiResponse( + { + ticketId: 'tkt-1', + source: 'discord', + pendingResponseRecovery: { messageId: 'msg-stale' }, + }, + makeContext({ jobId: 'job-takeover' }), + ); + + expect(result.data).toMatchObject({ + skipped: true, + escalated: true, + reason: 'delivery_recovered', + }); + expect(mockPrismaJob.create).toHaveBeenCalledTimes(1); + expect(mockPrismaJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ type: 'ESCALATION' }), + }); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + + it('atomically allows only one concurrent delayed recovery to enqueue escalation', async () => { + const pendingResponse = { + id: 'msg-concurrent-recovery', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-delayed-recovery', + responseError: 'Discord API 503', + createdAt: new Date(), + }; + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [...sampleTicket.messages, pendingResponse], + }); + + let pending = true; + mockPrismaMessage.updateMany.mockImplementation(async () => { + if (!pending) return { count: 0 }; + pending = false; + return { count: 1 }; + }); + + const payload = { + ticketId: 'tkt-1', + source: 'discord' as const, + pendingResponseRecovery: { messageId: pendingResponse.id }, + }; + const [first, second] = await Promise.all([ + handleAiResponse(payload, makeContext({ jobId: 'job-delayed-recovery' })), + handleAiResponse(payload, makeContext({ jobId: 'job-delayed-recovery' })), + ]); + + expect(first.success).toBe(true); + expect(second.success).toBe(true); + expect(mockPrismaJob.create).toHaveBeenCalledTimes(1); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + it('drives progress to 100 so the skipped job is not left looking hung', async () => { mockPrismaTicket.findUnique.mockResolvedValue(answeredTicket); const ctx = makeContext(); diff --git a/packages/outpost/queue/src/__tests__/queue.test.ts b/packages/outpost/queue/src/__tests__/queue.test.ts index 001bf882..275a73a3 100644 --- a/packages/outpost/queue/src/__tests__/queue.test.ts +++ b/packages/outpost/queue/src/__tests__/queue.test.ts @@ -19,11 +19,13 @@ import type { JobResult, JobHandlerContext, WorkerHealthStatus } from '../types. const mockPrismaJob = { create: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), findFirst: vi.fn(), }; const mockPrisma = { job: mockPrismaJob, + $executeRaw: vi.fn(), $queryRaw: vi.fn(), }; @@ -52,6 +54,7 @@ function makeJobRow( payload: unknown; attempts: number; maxAttempts: number; + claimToken: string; }> = {}, ) { return { @@ -60,6 +63,7 @@ function makeJobRow( payload: overrides.payload ?? { ticketId: 'tkt-1', source: 'discord' }, attempts: overrides.attempts ?? 0, maxAttempts: overrides.maxAttempts ?? 5, + claimToken: overrides.claimToken ?? 'claim-1', }; } @@ -120,31 +124,31 @@ describe('updateJobProgress', () => { }); it('updates progress clamped between 0 and 100', async () => { - mockPrismaJob.update.mockResolvedValue({}); + mockPrismaJob.updateMany.mockResolvedValue({ count: 1 }); - await updateJobProgress('job-1', 50); - expect(mockPrismaJob.update).toHaveBeenCalledWith({ - where: { id: 'job-1' }, + await updateJobProgress('job-1', 50, 'claim-1'); + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith({ + where: { id: 'job-1', status: 'PROCESSING', claimToken: 'claim-1' }, data: { progress: 50 }, }); }); it('clamps progress above 100 to 100', async () => { - mockPrismaJob.update.mockResolvedValue({}); + mockPrismaJob.updateMany.mockResolvedValue({ count: 1 }); - await updateJobProgress('job-1', 150); - expect(mockPrismaJob.update).toHaveBeenCalledWith({ - where: { id: 'job-1' }, + await updateJobProgress('job-1', 150, 'claim-1'); + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith({ + where: { id: 'job-1', status: 'PROCESSING', claimToken: 'claim-1' }, data: { progress: 100 }, }); }); it('clamps negative progress to 0', async () => { - mockPrismaJob.update.mockResolvedValue({}); + mockPrismaJob.updateMany.mockResolvedValue({ count: 1 }); - await updateJobProgress('job-1', -10); - expect(mockPrismaJob.update).toHaveBeenCalledWith({ - where: { id: 'job-1' }, + await updateJobProgress('job-1', -10, 'claim-1'); + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith({ + where: { id: 'job-1', status: 'PROCESSING', claimToken: 'claim-1' }, data: { progress: 0 }, }); }); @@ -156,6 +160,8 @@ describe('Worker', () => { beforeEach(() => { vi.clearAllMocks(); vi.useFakeTimers(); + mockPrisma.$executeRaw.mockResolvedValue(0); + mockPrismaJob.updateMany.mockResolvedValue({ count: 1 }); worker = new Worker({ pollIntervalMs: 100, maxConcurrency: 2, @@ -186,14 +192,157 @@ describe('Worker', () => { await vi.advanceTimersByTimeAsync(0); expect(results).toEqual(['tkt-1']); - expect(mockPrismaJob.update).toHaveBeenCalledWith( + const claimSql = mockPrisma.$queryRaw.mock.calls[0][0].join(' '); + expect(claimSql).toContain('"claimToken" = gen_random_uuid()::text'); + expect(claimSql).toContain('"maxAttempts", "claimToken"'); + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ - where: { id: 'job-1' }, + where: expect.objectContaining({ id: 'job-1', claimToken: 'claim-1' }), data: expect.objectContaining({ status: 'COMPLETED', attempts: 1 }), }), ); }); + it('reclaims stale processing jobs using each job type timeout before claiming', async () => { + const now = new Date('2026-08-11T12:00:00.000Z'); + vi.setSystemTime(now); + + await worker.stop(); + worker = new Worker({ + pollIntervalMs: 100, + maxConcurrency: 2, + defaultTimeoutMs: 5000, + jobTimeouts: { + [JobType.AI_RESPONSE]: 1000, + }, + }); + + const recoveredJob = makeJobRow(); + mockPrisma.$executeRaw.mockResolvedValue(1); + mockPrisma.$queryRaw.mockResolvedValueOnce([recoveredJob]); + mockPrisma.$queryRaw.mockResolvedValue([]); + mockPrismaJob.update.mockResolvedValue({}); + + const handled: string[] = []; + worker.on(JobType.AI_RESPONSE, async (payload) => { + handled.push(payload.ticketId); + return { success: true }; + }); + worker.on(JobType.ESCALATION, async () => ({ success: true })); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + + const firstReclaim = mockPrisma.$executeRaw.mock.calls[0]; + expect(JSON.parse(firstReclaim[1])).toEqual([ + { type: JobType.AI_RESPONSE, reclaim_after_ms: 31_000 }, + { type: JobType.ESCALATION, reclaim_after_ms: 35_000 }, + ]); + const reclaimSql = firstReclaim[0].join(' '); + expect(reclaimSql).toContain("WHERE job.status = 'PROCESSING'"); + expect(reclaimSql).toContain('job."lockedAt" < NOW()'); + expect(handled).toEqual(['tkt-1']); + }); + + it('does not let an old execution clobber the reclaimed claim', async () => { + const persisted = { + id: 'job-1', + status: 'PROCESSING', + claimToken: 'claim-old', + attempts: 0, + progress: null as number | null, + }; + mockPrismaJob.updateMany.mockImplementation(async ({ where, data }) => { + if ( + where.id !== persisted.id || + where.status !== persisted.status || + where.claimToken !== persisted.claimToken + ) { + return { count: 0 }; + } + Object.assign(persisted, data); + return { count: 1 }; + }); + + let oldStarted!: () => void; + const oldIsRunning = new Promise((resolve) => { + oldStarted = resolve; + }); + let releaseOld!: () => void; + const oldMayFinish = new Promise((resolve) => { + releaseOld = resolve; + }); + worker.on(JobType.AI_RESPONSE, async (payload, context) => { + if (payload.ticketId === 'old-execution') { + oldStarted(); + await oldMayFinish; + await context.reportProgress(25); + } + return { success: true }; + }); + + type ClaimedJob = ReturnType; + const processJob = ( + worker as unknown as { processJob(job: ClaimedJob): Promise } + ).processJob.bind(worker); + const oldExecution = processJob( + makeJobRow({ + payload: { ticketId: 'old-execution', source: 'discord' }, + claimToken: 'claim-old', + }), + ); + await oldIsRunning; + + // Model stale reclamation followed by a new exclusive claim. + persisted.claimToken = 'claim-new'; + persisted.attempts = 1; + const newExecution = processJob( + makeJobRow({ + payload: { ticketId: 'new-execution', source: 'discord' }, + attempts: 1, + claimToken: 'claim-new', + }), + ); + await newExecution; + + releaseOld(); + await oldExecution; + + expect(persisted).toMatchObject({ + status: 'COMPLETED', + claimToken: null, + attempts: 2, + progress: 100, + }); + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( + expect.objectContaining({ + where: { + id: 'job-1', + status: 'PROCESSING', + claimToken: 'claim-old', + }, + }), + ); + expect(mockPrismaJob.update).not.toHaveBeenCalled(); + }); + + it('counts crash-abandoned claims toward dead letter only after a recovery grace', async () => { + worker.on(JobType.AI_RESPONSE, async () => ({ success: true })); + mockPrisma.$queryRaw.mockResolvedValue([]); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + + const reclaimCall = mockPrisma.$executeRaw.mock.calls[0]; + expect(JSON.parse(reclaimCall[1])).toEqual([ + { type: JobType.AI_RESPONSE, reclaim_after_ms: 35_000 }, + ]); + const reclaimSql = reclaimCall[0].join(' '); + expect(reclaimSql).toContain('job."attempts" + 1'); + expect(reclaimSql).toContain("THEN 'DEAD_LETTER'"); + expect(reclaimSql).toContain('"claimToken" = NULL'); + }); + it('marks job DEAD_LETTER after maxAttempts exhausted', async () => { const jobRow = makeJobRow({ attempts: 4, maxAttempts: 5 }); // attempt will be 5 mockPrisma.$queryRaw.mockResolvedValueOnce([jobRow]); @@ -207,9 +356,9 @@ describe('Worker', () => { worker.start(); await vi.advanceTimersByTimeAsync(0); - expect(mockPrismaJob.update).toHaveBeenCalledWith( + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ - where: { id: 'job-1' }, + where: expect.objectContaining({ id: 'job-1', claimToken: 'claim-1' }), data: expect.objectContaining({ status: 'DEAD_LETTER', attempts: 5, @@ -232,9 +381,9 @@ describe('Worker', () => { worker.start(); await vi.advanceTimersByTimeAsync(0); - expect(mockPrismaJob.update).toHaveBeenCalledWith( + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ - where: { id: 'job-1' }, + where: expect.objectContaining({ id: 'job-1', claimToken: 'claim-1' }), data: expect.objectContaining({ status: 'PENDING', attempts: 2, @@ -268,7 +417,7 @@ describe('Worker', () => { await vi.advanceTimersByTimeAsync(200); // Should have been marked as retryable (attempt 1 of 5) - expect(mockPrismaJob.update).toHaveBeenCalledWith( + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ status: 'PENDING', @@ -288,7 +437,7 @@ describe('Worker', () => { worker.start(); await vi.advanceTimersByTimeAsync(0); - expect(mockPrismaJob.update).toHaveBeenCalledWith( + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ status: 'FAILED', @@ -329,7 +478,7 @@ describe('Worker', () => { await vi.advanceTimersByTimeAsync(100); // All 3 should complete (they run concurrently via Promise.allSettled) - expect(mockPrismaJob.update).toHaveBeenCalledTimes(3); + expect(mockPrismaJob.updateMany).toHaveBeenCalledTimes(3); }); it('provides accurate health check information', () => { @@ -374,6 +523,66 @@ describe('Worker', () => { expect(jobFinished).toBe(true); }); + it('shares the active-job drain across repeated stop calls', async () => { + let jobFinished = false; + const jobRow = makeJobRow(); + mockPrisma.$queryRaw.mockResolvedValueOnce([jobRow]); + mockPrisma.$queryRaw.mockResolvedValue([]); + mockPrismaJob.update.mockResolvedValue({}); + + worker.on(JobType.AI_RESPONSE, async () => { + await new Promise((resolve) => setTimeout(resolve, 200)); + jobFinished = true; + return { success: true }; + }); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + + const signalStop = worker.stop(); + let appStopResolved = false; + const appStop = worker.stop().then(() => { + appStopResolved = true; + }); + + await Promise.resolve(); + expect(appStopResolved).toBe(false); + expect(jobFinished).toBe(false); + + await vi.advanceTimersByTimeAsync(300); + await Promise.all([signalStop, appStop]); + + expect(jobFinished).toBe(true); + expect(appStopResolved).toBe(true); + }); + + it('waits for an in-flight poll and does not claim after shutdown begins', async () => { + let releaseReclaim!: () => void; + mockPrisma.$executeRaw.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseReclaim = () => resolve(0); + }), + ); + + worker.on(JobType.AI_RESPONSE, async () => ({ success: true })); + worker.start(); + + const stopPromise = worker.stop(); + let stopped = false; + void stopPromise.then(() => { + stopped = true; + }); + await Promise.resolve(); + expect(stopped).toBe(false); + + releaseReclaim(); + await stopPromise; + + expect(mockPrisma.$queryRaw).not.toHaveBeenCalled(); + expect(worker.healthCheck().running).toBe(false); + }); + it('handler receives context with progress reporting', async () => { const jobRow = makeJobRow(); mockPrisma.$queryRaw.mockResolvedValueOnce([jobRow]); @@ -393,8 +602,8 @@ describe('Worker', () => { expect(receivedContext).not.toBeNull(); expect(receivedContext!.jobId).toBe('job-1'); - // reportProgress should have called prisma.job.update with progress: 50 - const progressCall = mockPrismaJob.update.mock.calls.find( + // reportProgress should fence the update to this execution's claim. + const progressCall = mockPrismaJob.updateMany.mock.calls.find( (call: Array>>) => call[0].data.progress === 50, ); expect(progressCall).toBeDefined(); diff --git a/packages/outpost/queue/src/__tests__/worker-concurrency.test.ts b/packages/outpost/queue/src/__tests__/worker-concurrency.test.ts index 441c5f50..b708e989 100644 --- a/packages/outpost/queue/src/__tests__/worker-concurrency.test.ts +++ b/packages/outpost/queue/src/__tests__/worker-concurrency.test.ts @@ -16,11 +16,13 @@ import type { JobHandlerContext, WorkerHealthStatus } from '../types.js'; const mockPrismaJob = { create: vi.fn(), update: vi.fn(), + updateMany: vi.fn(), findFirst: vi.fn(), }; const mockPrisma = { job: mockPrismaJob, + $executeRaw: vi.fn(), $queryRaw: vi.fn(), }; @@ -45,6 +47,7 @@ function makeJobRow(overrides: Partial<{ payload: unknown; attempts: number; maxAttempts: number; + claimToken: string; }> = {}) { return { id: overrides.id ?? 'job-1', @@ -52,6 +55,7 @@ function makeJobRow(overrides: Partial<{ payload: overrides.payload ?? { ticketId: 'tkt-1', source: 'discord' }, attempts: overrides.attempts ?? 0, maxAttempts: overrides.maxAttempts ?? 5, + claimToken: overrides.claimToken ?? `claim-${overrides.id ?? 'job-1'}`, }; } @@ -63,6 +67,8 @@ describe('Worker per-type concurrency', () => { beforeEach(() => { vi.clearAllMocks(); vi.useFakeTimers(); + mockPrisma.$executeRaw.mockResolvedValue(0); + mockPrismaJob.updateMany.mockResolvedValue({ count: 1 }); }); afterEach(async () => { @@ -103,14 +109,44 @@ describe('Worker per-type concurrency', () => { // Process should have picked up the job expect(mockPrisma.$queryRaw).toHaveBeenCalled(); + const claimSql = mockPrisma.$queryRaw.mock.calls[0][0].join(' '); + expect(claimSql).toContain('"claimToken" = gen_random_uuid()::text'); + expect(claimSql).toContain('"maxAttempts", "claimToken"'); // Job should have been completed - expect(mockPrismaJob.update).toHaveBeenCalledWith( + expect(mockPrismaJob.updateMany).toHaveBeenCalledWith( expect.objectContaining({ data: expect.objectContaining({ status: 'COMPLETED' }), }), ); }); + it('reclaims stale processing jobs before per-type claims', async () => { + const now = new Date('2026-08-11T12:00:00.000Z'); + vi.setSystemTime(now); + worker = new Worker({ + pollIntervalMs: 100, + maxConcurrency: 2, + concurrencyByType: { + [JobType.AI_RESPONSE]: 1, + }, + jobTimeouts: { + [JobType.AI_RESPONSE]: 2000, + }, + }); + + mockPrisma.$queryRaw.mockResolvedValue([]); + worker.on(JobType.AI_RESPONSE, async () => ({ success: true })); + + worker.start(); + await vi.advanceTimersByTimeAsync(0); + + expect(mockPrisma.$executeRaw).toHaveBeenCalledTimes(1); + expect(JSON.parse(mockPrisma.$executeRaw.mock.calls[0][1])).toEqual([ + { type: JobType.AI_RESPONSE, reclaim_after_ms: 32_000 }, + ]); + expect(mockPrisma.$queryRaw).toHaveBeenCalled(); + }); + it('falls back to global limit when concurrencyByType is not specified', async () => { worker = new Worker({ pollIntervalMs: 100, @@ -187,7 +223,7 @@ describe('Worker per-type concurrency', () => { await vi.advanceTimersByTimeAsync(0); // Both AI jobs should have completed - const completedCalls = mockPrismaJob.update.mock.calls.filter( + const completedCalls = mockPrismaJob.updateMany.mock.calls.filter( (call: Array>>) => call[0].data.status === 'COMPLETED', ); diff --git a/packages/outpost/queue/src/create-job.ts b/packages/outpost/queue/src/create-job.ts index 59ac2f6c..266f8e4a 100644 --- a/packages/outpost/queue/src/create-job.ts +++ b/packages/outpost/queue/src/create-job.ts @@ -25,13 +25,17 @@ export async function createJob( } /** - * Update the progress of a running job. + * Update the progress of the running job claim that owns this execution. * Progress is a percentage from 0 to 100. */ -export async function updateJobProgress(jobId: string, percent: number): Promise { +export async function updateJobProgress( + jobId: string, + percent: number, + claimToken: string, +): Promise { const clamped = Math.max(0, Math.min(100, Math.round(percent))); - await prisma.job.update({ - where: { id: jobId }, + await prisma.job.updateMany({ + where: { id: jobId, status: 'PROCESSING', claimToken }, data: { progress: clamped }, }); } diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index 04469ec3..d617710e 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -17,11 +17,12 @@ * suppressed an ungrounded draft, or if the response never reached the * reporter because platform delivery failed * - * Delivery failure is escalated rather than swallowed because of the guard in - * step 1b (one response per ticket): once the BOT Message row exists, a retry or - * a manual re-enqueue is skipped, so an undelivered answer would otherwise leave - * the reporter permanently silent while the database claims they were answered. - * A human is the only remaining path, so the handler always pulls one in. + * The BOT Message starts in PENDING before any external post. Successful + * delivery with no human handoff marks it DELIVERED; a response that requires + * escalation stays PENDING until that job is durable, then becomes ESCALATED. + * If delivery itself ends PENDING, a retry schedules a delayed check. That + * check pulls in a human only if the response remains pending, preserving the + * one-post rule without racing the original handler. * * The pipeline itself handles Pathfinder retrieval, Claude generation, * confidence scoring, platform-specific formatting, and the groundedness gate — @@ -31,14 +32,247 @@ import { prisma } from '@copilotkit/outpost/db'; import { AIPipeline } from '@copilotkit/outpost/ai'; -import { AI_CONFIDENCE } from '@copilotkit/outpost/shared'; +import { AI_CONFIDENCE, MAX_JOB_ATTEMPTS } from '@copilotkit/outpost/shared'; import type { PlatformTarget, TicketSource } from '@copilotkit/outpost/shared'; import { hasAdapter, getAdapter } from '@copilotkit/outpost/shared/platforms'; -import { createJob } from '../create-job.js'; import { getFeedbackCalibration } from '../feedback-calibration.js'; import { JobType } from '../types.js'; import type { AiResponsePayload, JobResult, JobHandlerContext } from '../types.js'; +const PRIMARY_AI_RESPONSE_KEY = 'PRIMARY_AI_RESPONSE'; +const RESPONSE_RECOVERY_AFTER_MS = 5 * 60 * 1000; +const DELIVERY_CONFIRMED_MARKER = 'DELIVERY_CONFIRMED'; +const ESCALATION_REQUIRED_MARKER = 'ESCALATION_REQUIRED'; + +interface StoredAiResponse { + id: string; + type: string; + isAiGenerated: boolean; + responseKey?: string | null; + responseState?: string | null; + responseJobId?: string | null; + responseError?: string | null; +} + +/** + * Identify the unique-key collision raised when another handler wins the + * per-ticket primary-response slot. Keep this narrow: an unrelated P2002 must + * still fail the job rather than being mislabeled as a harmless duplicate. + */ +function isPrimaryAiResponseConflict(error: unknown): boolean { + if ( + typeof error !== 'object' || + error === null || + !('code' in error) || + (error as { code?: unknown }).code !== 'P2002' + ) { + return false; + } + + const target = (error as { meta?: { target?: unknown } }).meta?.target; + if (Array.isArray(target)) { + return target.includes('ticketId') && target.includes('responseKey'); + } + + return ( + typeof target === 'string' && + (target === 'Message_ticketId_responseKey_key' || + (target.includes('ticketId') && target.includes('responseKey'))) + ); +} + +function hasConfirmedDelivery(response: StoredAiResponse): boolean { + return response.responseError?.startsWith(`${DELIVERY_CONFIRMED_MARKER}:`) ?? false; +} + +/** + * Commit the PENDING -> ESCALATED transition and its queue row together. + * + * updateMany is the compare-and-set. PostgreSQL serializes concurrent updates + * to the same message row, so exactly one transaction observes PENDING. Creating + * the job after that CAS inside the same transaction means an insert failure + * rolls the state change back and leaves the response retryable. + */ +async function enqueueEscalationAtomically( + ticketId: string, + responseId: string, + reason: string, +): Promise { + return prisma.$transaction(async (tx) => { + const transition = await tx.message.updateMany({ + where: { + id: responseId, + responseKey: PRIMARY_AI_RESPONSE_KEY, + responseState: 'PENDING', + }, + data: { responseState: 'ESCALATED', responseError: null }, + }); + + if (transition.count !== 1) return false; + + await tx.job.create({ + data: { + type: JobType.ESCALATION, + payload: JSON.parse(JSON.stringify({ ticketId, reason })), + maxAttempts: MAX_JOB_ATTEMPTS, + // Omit runAt so the database's now() default, rather than the + // worker clock, makes the job immediately eligible. + }, + }); + return true; + }); +} + +function requiredEscalationReason(response: StoredAiResponse): string | null { + const prefix = `${ESCALATION_REQUIRED_MARKER}: `; + if ( + response.responseKey !== PRIMARY_AI_RESPONSE_KEY || + response.responseState !== 'PENDING' || + !response.responseError?.startsWith(prefix) + ) { + return null; + } + return response.responseError.slice(prefix.length); +} + +async function recoverRequiredEscalation( + ticketId: string, + response: StoredAiResponse, + reason: string, + context: JobHandlerContext, +): Promise { + let escalationEnqueued: boolean; + try { + escalationEnqueued = await enqueueEscalationAtomically(ticketId, response.id, reason); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + success: false, + error: `Ticket ${ticketId}: required escalation retry could not enqueue (${message})`, + }; + } + + await context.reportProgress(100); + return { + success: true, + data: { + ticketId, + skipped: true, + escalated: escalationEnqueued, + deliveryFailed: false, + reason: 'escalation_recovered', + }, + }; +} + +async function recoverPendingResponse( + ticketId: string, + ticketSource: string, + response: StoredAiResponse, + context: JobHandlerContext, +): Promise { + const deliveryDetail = response.responseError + ? `Last delivery error: ${response.responseError}.` + : 'The previous attempt ended before delivery became durable.'; + const reason = + `AI response for ${ticketSource} is pending after an interrupted attempt. ` + + `${deliveryDetail} A human must verify the thread and answer if needed.`; + + let escalationEnqueued: boolean; + try { + escalationEnqueued = await enqueueEscalationAtomically(ticketId, response.id, reason); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + success: false, + error: `Ticket ${ticketId}: pending AI response recovery could not enqueue escalation (${message})`, + }; + } + + await context.reportProgress(100); + return { + success: true, + data: { + ticketId, + skipped: true, + escalated: escalationEnqueued, + deliveryFailed: true, + reason: 'delivery_recovered', + }, + }; +} + +async function schedulePendingResponseRecovery( + payload: AiResponsePayload, + response: StoredAiResponse, + context: JobHandlerContext, +): Promise { + let recoveryJobId: string; + try { + recoveryJobId = await prisma.$transaction(async (tx) => { + const [clock] = await tx.$queryRaw>` + SELECT CURRENT_TIMESTAMP AS "now" + `; + if (!clock) throw new Error('database clock unavailable'); + + const recoveryPayload: AiResponsePayload = { + ...payload, + pendingResponseRecovery: { messageId: response.id }, + }; + const recoveryJob = await tx.job.create({ + data: { + type: JobType.AI_RESPONSE, + payload: JSON.parse(JSON.stringify(recoveryPayload)), + maxAttempts: MAX_JOB_ATTEMPTS, + runAt: new Date(clock.now.getTime() + RESPONSE_RECOVERY_AFTER_MS), + }, + }); + + const transfer = await tx.message.updateMany({ + where: { + id: response.id, + responseKey: PRIMARY_AI_RESPONSE_KEY, + responseState: 'PENDING', + responseJobId: context.jobId, + }, + data: { responseJobId: recoveryJob.id }, + }); + if (transfer.count !== 1) { + // Throwing rolls the just-created recovery job back too. + throw new PendingRecoveryClaimLostError(); + } + return recoveryJob.id; + }); + } catch (error) { + if (error instanceof PendingRecoveryClaimLostError) { + await context.reportProgress(100); + return { + success: true, + data: { ticketId: payload.ticketId, skipped: true, reason: 'already_answered' }, + }; + } + const message = error instanceof Error ? error.message : String(error); + return { + success: false, + error: `Ticket ${payload.ticketId}: delayed AI response recovery could not be scheduled (${message})`, + }; + } + + await context.reportProgress(100); + return { + success: true, + data: { + ticketId: payload.ticketId, + skipped: true, + recoveryScheduled: true, + recoveryJobId, + reason: 'delivery_recovery_scheduled', + }, + }; +} + +class PendingRecoveryClaimLostError extends Error {} + /** * Map from TicketSource enum values (stored in DB) to PlatformTarget * strings used by the AI formatter. TicketSource uses uppercase enums @@ -118,10 +352,64 @@ export async function handleAiResponse( // Success, not failure: the job did what it should — nothing. Returning an // error would put it through the retry ladder for a decision that will // never change. - const priorAiResponse = ticket.messages.find( - (m: { type: string; isAiGenerated: boolean }) => m.type === 'BOT' && m.isAiGenerated, - ); + const generatedResponses = ticket.messages.filter( + (m: StoredAiResponse) => m.type === 'BOT' && m.isAiGenerated, + ) as StoredAiResponse[]; + // Recovery metadata lives on the keyed primary response. Older AI BOT rows + // predate responseKey and still prove the ticket was answered, but must not + // shadow a newer primary row whose delivery/escalation state needs repair. + const priorAiResponse = + generatedResponses.find((m) => m.responseKey === PRIMARY_AI_RESPONSE_KEY) ?? + generatedResponses[0]; if (priorAiResponse) { + if (priorAiResponse.responseState === 'PENDING' && hasConfirmedDelivery(priorAiResponse)) { + // The platform post succeeded; only the state mirror failed. Repair + // it when possible, but never route an already-answered reporter to + // a human merely because this bookkeeping write is still unhealthy. + try { + await prisma.message.update({ + where: { id: priorAiResponse.id }, + data: { responseState: 'DELIVERED', responseError: null }, + }); + } catch (error) { + console.error( + `[AI Response] Confirmed delivery state still could not be repaired for ticket ${ticketId}:`, + error instanceof Error ? error.message : String(error), + ); + } + await context.reportProgress(100); + return { + success: true, + data: { ticketId, skipped: true, reason: 'already_answered' }, + }; + } + + const escalationRetryReason = requiredEscalationReason(priorAiResponse); + if (escalationRetryReason) { + return recoverRequiredEscalation( + ticketId, + priorAiResponse, + escalationRetryReason, + context, + ); + } + + if ( + priorAiResponse.responseKey === PRIMARY_AI_RESPONSE_KEY && + priorAiResponse.responseState === 'PENDING' && + payload.pendingResponseRecovery?.messageId === priorAiResponse.id && + priorAiResponse.responseJobId === context.jobId + ) { + return recoverPendingResponse(ticketId, ticket.source, priorAiResponse, context); + } + if ( + priorAiResponse.responseKey === PRIMARY_AI_RESPONSE_KEY && + priorAiResponse.responseState === 'PENDING' && + priorAiResponse.responseJobId === context.jobId + ) { + return schedulePendingResponseRecovery(payload, priorAiResponse, context); + } + console.log( `[AI Response] Ticket ${ticketId} already answered — skipping. ` + `Outpost posts one response per ticket; a human owns this thread now.`, @@ -139,21 +427,22 @@ export async function handleAiResponse( }; } - // 2. Build conversation history from DB messages. + // The question is the message that OPENED the ticket — the same message the + // one-response-per-ticket invariant above says we get to answer. + const openingUserMessage = ticket.messages.find((m: { type: string }) => m.type === 'USER'); + + // 2. Build conversation context from every other non-SYSTEM message. // - // Deliberately the FULL non-SYSTEM history, including any message that - // arrived after the one being answered. See the question selection below for - // why the two are allowed to disagree. + // AIPipeline ultimately appends `question` after `conversationHistory`, so + // including the opening row here would send that question twice. Keep later + // follow-ups as context, but let the explicit question carry the opener once. const conversationHistory = ticket.messages - .filter((m: { type: string }) => m.type !== 'SYSTEM') + .filter((m: { type: string }) => m.type !== 'SYSTEM' && m !== openingUserMessage) .map((m: { type: string; content: string }) => ({ role: (m.type === 'USER' ? 'user' : 'assistant') as 'user' | 'assistant', content: m.content, })); - // The question is the message that OPENED the ticket — the same message the - // one-response-per-ticket invariant above says we get to answer. - // // `ticket.messages` is loaded `orderBy: { createdAt: 'asc' }`, so the FIRST // USER row is the opening message. Scanning from the other end and taking // the LATEST USER row was wrong: replies are still persisted as USER @@ -171,7 +460,6 @@ export async function handleAiResponse( // dropping it would trade one bug for a worse answer. Suppressing it would // also need a second policy for the non-USER rows after the opening, with no // evidence behind it. - const openingUserMessage = ticket.messages.find((m: { type: string }) => m.type === 'USER'); const question = openingUserMessage?.content ?? ticket.description ?? ticket.title; // Determine platform target for formatting @@ -201,13 +489,15 @@ export async function handleAiResponse( console.log(`[AI Response] Confidence calibration: ${confidenceCalibration.toFixed(4)}`); // Why the response never reached the reporter, when it didn't. Set by the - // post-back arm below and consumed by the escalation step: the one-response- - // per-ticket guard makes an undelivered answer unrecoverable by retry, so a - // human has to take the thread. + // post-back arm below and consumed by the escalation step. If this attempt + // cannot durably enqueue that escalation, the PENDING response lets its + // retry schedule a safe delayed handoff without risking a second post. let deliveryFailure: string | null = null; - // Set when the ESCALATION enqueue itself failed after a delivery failure — - // the one case where the job must not report success (see the return below). + // Set when any required ESCALATION enqueue fails. Delivery failures, + // suppression, and low confidence all promise a human handoff, so none may + // report success until that handoff is durable. let escalationEnqueueError: string | null = null; + let escalationReason: string | null = null; let pipelineResult; try { @@ -249,26 +539,49 @@ export async function handleAiResponse( await context.reportProgress(70); - // 5. Persist the AI-generated response as a Message record - const aiMessage = await prisma.message.create({ - data: { + // 5. Persist the AI-generated response and atomically claim this + // ticket's one primary-response slot. The history check above avoids + // unnecessary model work in the common case, but it cannot serialize + // overlapping jobs: both can read the same no-response snapshot. The + // database unique key on (ticketId, responseKey) elects exactly one + // winner before either invocation reaches platform post-back. + let aiMessage; + try { + const aiMessageData = { ticketId: ticket.id, content: pipelineResult.response, - type: 'BOT', + type: 'BOT' as const, author: 'Outpost AI', isAiGenerated: true, confidenceScore: pipelineResult.confidenceScore, confidenceLevel: pipelineResult.confidenceLevel, - }, - }); + responseKey: PRIMARY_AI_RESPONSE_KEY, + responseState: 'PENDING' as const, + responseJobId: context.jobId, + responseError: null, + }; + aiMessage = await prisma.message.create({ + data: aiMessageData, + }); + } catch (error) { + if (!isPrimaryAiResponseConflict(error)) throw error; + + console.log( + `[AI Response] Ticket ${ticketId} was answered by a concurrent job — skipping platform post-back.`, + ); + await context.reportProgress(100); + return { + success: true, + data: { ticketId, skipped: true, reason: 'already_answered' }, + }; + } // Store the formatted response on the ticket for bots to pick up. // // Non-fatal on purpose. The BOT Message row is already committed above, - // which arms the one-response-per-ticket guard — so if this write threw, - // the job would abort before post-back and every retry would be skipped - // by that guard, leaving the reporter permanently unanswered. Log it, - // remember it, and keep going so delivery still happens. + // so aborting here would turn the retry into delayed human recovery + // rather than giving this attempt the chance to complete its intended + // delivery. Log it, remember it, and keep going so delivery can happen. let suggestedResponseError: string | null = null; try { await prisma.ticket.update({ @@ -306,6 +619,7 @@ export async function handleAiResponse( ); } + let responseDelivered = false; if (process.env.SHADOW_MODE === 'true') { try { await prisma.message.create({ @@ -331,6 +645,9 @@ export async function handleAiResponse( error instanceof Error ? error.message : String(error), ); } + // Shadow mode's intended sink is the SYSTEM row. Preserve its + // historical fail-soft behavior even if that diagnostic write fails. + responseDelivered = true; } else if (hasAdapter(ticketSource)) { let adapter; try { @@ -361,6 +678,7 @@ export async function handleAiResponse( console.log( `[AI Response] Posted response to ${ticket.source} for ticket ${ticketId}`, ); + responseDelivered = true; } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error( @@ -387,10 +705,85 @@ export async function handleAiResponse( } } } - } else if (suggestedResponseError) { - // No adapter for this source, so suggestedResponse WAS the delivery - // path — and that write failed. Nothing reached the reporter. - deliveryFailure = `no platform adapter for ${ticket.source} and suggestedResponse could not be stored: ${suggestedResponseError}`; + } else { + // For sources without adapters, suggestedResponse is the durable sink. + if (suggestedResponseError) { + deliveryFailure = `no platform adapter for ${ticket.source} and suggestedResponse could not be stored: ${suggestedResponseError}`; + } else { + responseDelivered = true; + } + } + + const nonDeliveryEscalationReason = pipelineResult.suppressed + ? `AI response withheld (${pipelineResult.groundedness.reasons.join('; ')}) — needs a human answer` + : pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE + ? `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation` + : null; + + if (responseDelivered) { + if (nonDeliveryEscalationReason) { + // Keep the response PENDING until its promised human handoff is + // durable. A failed enqueue then retries this reason through the + // prior-response gate without regenerating or reposting. + try { + await prisma.message.update({ + where: { id: aiMessage.id }, + data: { + responseError: `${ESCALATION_REQUIRED_MARKER}: ${nonDeliveryEscalationReason}`, + }, + }); + } catch (error) { + console.error( + `[AI Response] Failed to record required escalation for ticket ${ticketId}:`, + error instanceof Error ? error.message : String(error), + ); + } + } else { + try { + await prisma.message.update({ + where: { id: aiMessage.id }, + data: { responseState: 'DELIVERED', responseError: null }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error( + `[AI Response] Failed to record durable delivery for ticket ${ticketId}:`, + message, + ); + // Delivery is already a fact. Persist a separate confirmation + // marker so a retry can repair the state without reposting or + // escalating an already-answered reporter. + try { + await prisma.message.update({ + where: { id: aiMessage.id }, + data: { + responseError: `${DELIVERY_CONFIRMED_MARKER}: ${message}`, + }, + }); + } catch (markerError) { + console.error( + `[AI Response] Failed to record delivery confirmation marker for ticket ${ticketId}:`, + markerError instanceof Error + ? markerError.message + : String(markerError), + ); + } + } + } + } + + if (deliveryFailure) { + try { + await prisma.message.update({ + where: { id: aiMessage.id }, + data: { responseError: deliveryFailure }, + }); + } catch (error) { + console.error( + `[AI Response] Failed to record delivery error for ticket ${ticketId}:`, + error instanceof Error ? error.message : String(error), + ); + } } await context.reportProgress(85); @@ -399,22 +792,21 @@ export async function handleAiResponse( // was withheld, or when confidence is below threshold — in the first two // cases nothing useful reached the reporter, so a human has to pick it up // regardless of what the score says. Delivery failure wins the reason slot - // because it is the most actionable: the answer exists but is undelivered - // and, thanks to the one-response-per-ticket guard, undeliverable by retry. - const escalationReason = deliveryFailure + // because it is the most actionable: the answer exists but is undelivered. + // A stale-recovery job will escalate a response left PENDING, never + // post it again. + escalationReason = deliveryFailure ? `AI response generated but not delivered to ${ticket.source} (${deliveryFailure}) — needs a human to answer the reporter` - : pipelineResult.suppressed - ? `AI response withheld (${pipelineResult.groundedness.reasons.join('; ')}) — needs a human answer` - : pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE - ? `Low AI confidence (${(pipelineResult.confidenceScore * 100).toFixed(0)}%) — automated escalation` - : null; + : nonDeliveryEscalationReason; + let escalationEnqueued = false; if (escalationReason) { try { - await createJob(JobType.ESCALATION, { - ticketId: ticket.id, - reason: escalationReason, - }); + escalationEnqueued = await enqueueEscalationAtomically( + ticket.id, + aiMessage.id, + escalationReason, + ); } catch (error) { escalationEnqueueError = error instanceof Error ? error.message : String(error); console.error( @@ -427,8 +819,6 @@ export async function handleAiResponse( pipeline.destroy(); } - await context.reportProgress(100); - console.log( `[AI Response] Ticket ${ticketId}: confidence=${pipelineResult.confidenceLevel} ` + `(${(pipelineResult.confidenceScore * 100).toFixed(0)}%), latency=${pipelineResult.latencyMs}ms` + @@ -441,21 +831,22 @@ export async function handleAiResponse( pipelineResult.suppressed || pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE; - // An undelivered answer with no escalation behind it is the one outcome that - // leaves the reporter silent and no human involved, and the guard blocks any - // retry from repairing it. Report failure so the attempt is recorded as failed - // and surfaces to an operator rather than being logged and forgotten. Other - // escalation-enqueue failures keep the historical success result: in those the - // response did reach the reporter. - if (deliveryFailure && escalationEnqueueError) { + // A promised human handoff is part of successful completion even when the AI + // response reached the reporter. Report enqueue failure so the queue retries: + // delivered low-confidence/suppressed responses carry their reason through + // the prior-response gate, while delivery failures use the pending-response + // recovery path. Neither route posts the AI response again. + if (escalationReason && escalationEnqueueError) { return { success: false, error: - `Ticket ${ticketId}: AI response not delivered (${deliveryFailure}) and ` + - `escalation could not be enqueued (${escalationEnqueueError}) — needs manual attention`, + `Ticket ${ticketId}: required escalation (${escalationReason}) ` + + `could not be enqueued (${escalationEnqueueError}) — needs manual attention`, }; } + await context.reportProgress(100); + return { success: true, data: { diff --git a/packages/outpost/queue/src/types.ts b/packages/outpost/queue/src/types.ts index fc9ce17a..53e31ea1 100644 --- a/packages/outpost/queue/src/types.ts +++ b/packages/outpost/queue/src/types.ts @@ -38,6 +38,13 @@ export interface AiResponsePayload { ticketId: string; threadId?: string; source?: PlatformTarget; + /** + * Durable authorization for a delayed PENDING-response takeover. The + * message ID and the job's ownership row replace worker-clock age checks. + */ + pendingResponseRecovery?: { + messageId: string; + }; } export interface TicketClassifyPayload { diff --git a/packages/outpost/queue/src/worker.ts b/packages/outpost/queue/src/worker.ts index 6c7738ad..5c6020ed 100644 --- a/packages/outpost/queue/src/worker.ts +++ b/packages/outpost/queue/src/worker.ts @@ -10,6 +10,17 @@ import type { JobHandlerContext, } from './types.js'; +const STALE_RECOVERY_GRACE_MS = 30_000; + +interface ClaimedJob { + id: string; + type: string; + payload: unknown; + attempts: number; + maxAttempts: number; + claimToken: string; +} + /** * A worker that polls the Postgres job queue and processes jobs using * SELECT ... FOR UPDATE SKIP LOCKED for safe concurrent processing. @@ -33,12 +44,14 @@ export class Worker { private jobTimeouts: Partial>; private defaultTimeoutMs: number; private pollTimer: ReturnType | null = null; + private pollPromise: Promise | null = null; private activeJobs = new Set(); /** Track active job counts per type for per-type concurrency enforcement */ private activeJobsByType = new Map(); private lastPollTime: Date | null = null; private upSince: Date | null = null; private shutdownResolve: (() => void) | null = null; + private stopPromise: Promise | null = null; private signalHandlers: { signal: string; handler: () => void }[] = []; constructor(options?: WorkerOptions) { @@ -65,10 +78,11 @@ export class Worker { if (this.running) return; this.running = true; this.shuttingDown = false; + this.stopPromise = null; this.upSince = new Date(); console.log('[Queue Worker] Started'); this.registerSignalHandlers(); - this.poll(); + this.runPoll(); } /** @@ -76,6 +90,11 @@ export class Worker { * Waits for all active jobs to complete before resolving. */ async stop(): Promise { + // Signal handlers and the worker app can both request shutdown. Share + // the same drain promise so a second caller cannot observe + // `running=false`, return early, and disconnect Prisma/exit while the + // first caller is still waiting for active jobs. + if (this.stopPromise) return this.stopPromise; if (!this.running) return; this.shuttingDown = true; this.running = false; @@ -87,21 +106,30 @@ export class Worker { this.removeSignalHandlers(); - // Wait for active jobs to finish - if (this.activeJobs.size > 0) { - console.log(`[Queue Worker] Waiting for ${this.activeJobs.size} active jobs to complete...`); - await new Promise((resolve) => { - this.shutdownResolve = resolve; - // Check immediately in case jobs finished between the check and setting the resolver - if (this.activeJobs.size === 0) { - this.shutdownResolve = null; - resolve(); - } - }); - } + this.stopPromise = (async () => { + // A poll may be between its running check and its atomic claim. Let + // that cycle finish before deciding whether the active set is + // drained, otherwise stop() can resolve just before it claims work. + await this.pollPromise; + + // Wait for active jobs to finish + if (this.activeJobs.size > 0) { + console.log(`[Queue Worker] Waiting for ${this.activeJobs.size} active jobs to complete...`); + await new Promise((resolve) => { + this.shutdownResolve = resolve; + // Check immediately in case jobs finished between the check and setting the resolver + if (this.activeJobs.size === 0) { + this.shutdownResolve = null; + resolve(); + } + }); + } - this.upSince = null; - console.log('[Queue Worker] Stopped'); + this.upSince = null; + console.log('[Queue Worker] Stopped'); + })(); + + return this.stopPromise; } /** @@ -136,6 +164,19 @@ export class Worker { this.signalHandlers = []; } + private runPoll(): void { + const currentPoll = this.poll(); + this.pollPromise = currentPoll; + void currentPoll.finally(() => { + if (this.pollPromise === currentPoll) this.pollPromise = null; + }); + } + + private schedulePoll(delayMs: number): void { + if (!this.running) return; + this.pollTimer = setTimeout(() => this.runPoll(), delayMs); + } + private async poll(): Promise { if (!this.running) return; @@ -145,10 +186,13 @@ export class Worker { if (availableSlots <= 0) { // At capacity, wait and retry - this.pollTimer = setTimeout(() => this.poll(), this.pollIntervalMs); + this.schedulePoll(this.pollIntervalMs); return; } + await this.reclaimStaleJobs(); + if (!this.running) return; + const hasPerTypeLimits = Object.keys(this.concurrencyByType).length > 0; let processedCount: number; @@ -162,13 +206,62 @@ export class Worker { // If we processed jobs, poll immediately for more const nextPollDelay = processedCount > 0 ? 0 : this.pollIntervalMs; - this.pollTimer = setTimeout(() => this.poll(), nextPollDelay); + this.schedulePoll(nextPollDelay); } catch (error) { console.error('[Queue Worker] Poll error:', error); - this.pollTimer = setTimeout(() => this.poll(), this.pollIntervalMs); + this.schedulePoll(this.pollIntervalMs); } } + /** + * Return abandoned PROCESSING jobs to the pending queue before claiming work. + * + * lockedAt is written with the database clock, so the stale comparison must + * also use the database clock. Each registered type gets its own handler + * timeout plus a recovery grace: the normal timeout path must have time to + * release its claim before another worker calls it crash-abandoned. A true + * abandonment consumes an attempt, clears its claim token, and moves toward + * DEAD_LETTER like every other failed execution. + */ + private async reclaimStaleJobs(): Promise { + const policies = Array.from(this.handlers.keys(), (type) => ({ + type, + reclaim_after_ms: + (this.jobTimeouts[type as JobType] ?? this.defaultTimeoutMs) + + STALE_RECOVERY_GRACE_MS, + })); + + if (policies.length === 0) return; + + await prisma.$executeRaw` + UPDATE "Job" AS job + SET status = CASE + WHEN job."attempts" + 1 >= job."maxAttempts" + THEN 'DEAD_LETTER'::"JobStatus" + ELSE 'PENDING'::"JobStatus" + END, + "attempts" = job."attempts" + 1, + "lockedAt" = NULL, + "claimToken" = NULL, + progress = NULL, + error = 'Worker claim was abandoned before completion', + "completedAt" = CASE + WHEN job."attempts" + 1 >= job."maxAttempts" THEN NOW() + ELSE NULL + END, + "runAt" = CASE + WHEN job."attempts" + 1 >= job."maxAttempts" THEN job."runAt" + ELSE NOW() + END, + "updatedAt" = NOW() + FROM jsonb_to_recordset(${JSON.stringify(policies)}::jsonb) + AS policy(type text, reclaim_after_ms double precision) + WHERE job.status = 'PROCESSING' + AND job.type = policy.type + AND job."lockedAt" < NOW() - (policy.reclaim_after_ms * INTERVAL '1 millisecond') + `; + } + /** * Claim jobs respecting per-type concurrency limits. * For each registered job type that has available capacity, claim up to @@ -228,26 +321,11 @@ export class Worker { private async claimJobsForType( type: string, limit: number, - ): Promise< - Array<{ - id: string; - type: string; - payload: unknown; - attempts: number; - maxAttempts: number; - }> - > { - return prisma.$queryRaw< - Array<{ - id: string; - type: string; - payload: unknown; - attempts: number; - maxAttempts: number; - }> - >` + ): Promise> { + return prisma.$queryRaw>` UPDATE "Job" - SET status = 'PROCESSING', "lockedAt" = NOW(), "updatedAt" = NOW() + SET status = 'PROCESSING', "lockedAt" = NOW(), + "claimToken" = gen_random_uuid()::text, "updatedAt" = NOW() WHERE id IN ( SELECT id FROM "Job" WHERE status = 'PENDING' @@ -257,24 +335,17 @@ export class Worker { LIMIT ${limit} FOR UPDATE SKIP LOCKED ) - RETURNING id, type, payload, attempts, "maxAttempts" + RETURNING id, type, payload, attempts, "maxAttempts", "claimToken" `; } private async claimAndProcessJobs(limit: number): Promise { // Use raw query with SKIP LOCKED for safe concurrent job processing. // This atomically selects and locks pending jobs that are ready to run. - const jobs = await prisma.$queryRaw< - Array<{ - id: string; - type: string; - payload: unknown; - attempts: number; - maxAttempts: number; - }> - >` + const jobs = await prisma.$queryRaw>` UPDATE "Job" - SET status = 'PROCESSING', "lockedAt" = NOW(), "updatedAt" = NOW() + SET status = 'PROCESSING', "lockedAt" = NOW(), + "claimToken" = gen_random_uuid()::text, "updatedAt" = NOW() WHERE id IN ( SELECT id FROM "Job" WHERE status = 'PENDING' @@ -283,23 +354,17 @@ export class Worker { LIMIT ${limit} FOR UPDATE SKIP LOCKED ) - RETURNING id, type, payload, attempts, "maxAttempts" + RETURNING id, type, payload, attempts, "maxAttempts", "claimToken" `; // Process jobs concurrently (each tracked in activeJobs) - const promises = jobs.map((job: { id: string; type: string; payload: unknown; attempts: number; maxAttempts: number }) => this.processJob(job)); + const promises = jobs.map((job) => this.processJob(job)); await Promise.allSettled(promises); return jobs.length; } - private async processJob(job: { - id: string; - type: string; - payload: unknown; - attempts: number; - maxAttempts: number; - }): Promise { + private async processJob(job: ClaimedJob): Promise { this.activeJobs.add(job.id); this.activeJobsByType.set( job.type, @@ -311,12 +376,18 @@ export class Worker { if (!handler) { console.warn(`[Queue Worker] No handler for job type: ${job.type}`); - await prisma.job.update({ - where: { id: job.id }, + await prisma.job.updateMany({ + where: { + id: job.id, + status: 'PROCESSING', + claimToken: job.claimToken, + }, data: { status: 'FAILED', error: `No handler registered for job type: ${job.type}`, completedAt: new Date(), + lockedAt: null, + claimToken: null, }, }); return; @@ -328,7 +399,8 @@ export class Worker { // Build handler context const context: JobHandlerContext = { jobId: job.id, - reportProgress: (percent: number) => updateJobProgress(job.id, percent), + reportProgress: (percent: number) => + updateJobProgress(job.id, percent, job.claimToken), }; try { @@ -338,22 +410,39 @@ export class Worker { ); if (result.success) { - await prisma.job.update({ - where: { id: job.id }, + await prisma.job.updateMany({ + where: { + id: job.id, + status: 'PROCESSING', + claimToken: job.claimToken, + }, data: { status: 'COMPLETED', attempts: attempt, progress: 100, completedAt: new Date(), lockedAt: null, + claimToken: null, }, }); } else { - await this.handleFailure(job.id, attempt, job.maxAttempts, result.error ?? 'Unknown error'); + await this.handleFailure( + job.id, + job.claimToken, + attempt, + job.maxAttempts, + result.error ?? 'Unknown error', + ); } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - await this.handleFailure(job.id, attempt, job.maxAttempts, errorMessage); + await this.handleFailure( + job.id, + job.claimToken, + attempt, + job.maxAttempts, + errorMessage, + ); } } finally { this.activeJobs.delete(job.id); @@ -386,45 +475,52 @@ export class Worker { private async handleFailure( jobId: string, + claimToken: string, attempt: number, maxAttempts: number, error: string, ): Promise { if (attempt >= maxAttempts) { // Dead letter: job has exhausted all retries - await prisma.job.update({ - where: { id: jobId }, + const result = await prisma.job.updateMany({ + where: { id: jobId, status: 'PROCESSING', claimToken }, data: { status: 'DEAD_LETTER', attempts: attempt, error, completedAt: new Date(), lockedAt: null, + claimToken: null, }, }); - console.error( - `[Queue Worker] Job ${jobId} moved to dead letter queue after ${attempt} attempts: ${error}`, - ); + if (result.count > 0) { + console.error( + `[Queue Worker] Job ${jobId} moved to dead letter queue after ${attempt} attempts: ${error}`, + ); + } } else { // Schedule retry with exponential backoff const backoffMs = calculateBackoff(attempt); const runAt = new Date(Date.now() + backoffMs); - await prisma.job.update({ - where: { id: jobId }, + const result = await prisma.job.updateMany({ + where: { id: jobId, status: 'PROCESSING', claimToken }, data: { status: 'PENDING', attempts: attempt, error, runAt, lockedAt: null, + claimToken: null, progress: null, }, }); - console.warn( - `[Queue Worker] Job ${jobId} failed (attempt ${attempt}/${maxAttempts}), ` + - `retrying at ${runAt.toISOString()}: ${error}`, - ); + if (result.count > 0) { + console.warn( + `[Queue Worker] Job ${jobId} failed (attempt ${attempt}/${maxAttempts}), ` + + `retrying at ${runAt.toISOString()}: ${error}`, + ); + } } } } From f17f140d214cfe4c6cad1a68200f5892345c4866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:28:10 -0400 Subject: [PATCH 83/83] fix(queue,web): close the three blockers on the delivery-durability delta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The split brief named three findings that had to be fixed before this work could merge. All three lived in the half of PR #170 that never had a review pass. 1. escalationEnqueued was assigned and never read on the main path. enqueueEscalationAtomically returns false when its compare-and-set found the response row in a state other than PENDING, meaning no escalation was queued. That false was discarded: nothing logged, and the handler returned success with `escalated` computed from local conditions rather than from what actually happened. A response that reached nobody and summoned nobody reported itself handled. A no-op CAS now reads the row's actual responseState and splits the two cases that were previously identical. DELIVERED means no escalation was needed, so the job succeeds honestly with escalated: false. ESCALATED means someone else already queued it, so escalated: true. PENDING, missing, or unreadable means the handoff did not happen and the job fails loudly. `escalated` is now derived from the durable handoff instead of inferred. 2. Header-based email reply resolution trusted attacker-controlled input. findTicketByReplyMessageIds matched candidate IDs from In-Reply-To and References against Ticket.sourceId and Message.attachments.postmarkMessageId. Those headers are sender-supplied, and a Message-ID from a thread is KNOWN to every participant who was ever CC'd rather than having to be guessed. Anyone holding one could append a message to that ticket and reopen it. No AI response is spent on a reply and nothing is echoed back, so this was integrity and noise rather than disclosure — but materially wider than the MailboxHash path it supplements. Header matching now additionally requires the sender to be an existing participant: ticket.user.email, or an address parsed out of an existing Message.author on that ticket, or anyone at ticket.account.domain. The Message.author path is load-bearing rather than defensive — email tickets never set userId, so the opening sender is identified only by the "Name " message label. Deriving a domain from user.email or a message author was considered and rejected: a ticket opened from an address at a freemail provider would make every address at that provider a participant, which reopens the hole while looking like a fix. Account.domain is opt-in CRM data and cannot silently widen to a public provider. normalizeParticipantEmail also keeps free-form author values that are not addresses (System, slack:U123, Outpost AI) from counting as identities. An unauthorized candidate is treated as unresolved, so the mail falls through to the orphaned-reply path and is filed as its own ticket with no AI job. A legitimate sender writing from an unrecognised address never loses their message. MailboxHash is deliberately NOT gated — it is a plus-addressed token we generate, and gating it breaks the legitimate reply path. Two supporting changes the fix required: findFirst became findMany so candidates are scanned oldest-first for the first AUTHORIZED match, since otherwise a chain naming somebody else's older ticket would refuse the sender's own legitimate reply; and MAX_REPLY_MESSAGE_IDS caps the sender-supplied IN (...) fan-out that requiring participants necessarily widens. 3. responseError carried lifecycle state in a free-text error column. DELIVERY_CONFIRMED: and ESCALATION_REQUIRED: string prefixes encoded control state in the column an operator reads to find out what went wrong — so any ops surface would render DELIVERY_CONFIRMED as an error. In the same change that introduced MessageResponseState for exactly this purpose. Both moved to dedicated additive columns: deliveryConfirmed Boolean @default(false) and escalationRequiredReason String?. Columns rather than new enum values, deliberately: both markers describe a response that is still PENDING, and MessageResponseState records the OUTCOME, so adding them would have destroyed the outcome they are sub-states of and silently broken every responseState === 'PENDING' comparison in the handler. The escalation signal is one nullable text column rather than a boolean plus a reason, so the fact and its payload cannot drift apart. responseError now holds only real error text. Migration 20260813000000_add_message_response_substates is two ADD COLUMNs, no index, and needs no backfill: DEFAULT false makes existing rows correct by construction, and a row that somehow did hold a marker reads as "not confirmed", which is the safe direction. Verified against the merged CI drift gate by having Prisma 6.19.3 emit the required SQL via migrate diff and comparing it to the hand-written file; they are identical. No Postgres in the sandbox, so CI's own migrate deploy round-trip was not executed locally. One deliberate behaviour change falls out of (3): independent columns can both be set, which a single string prefix made impossible. The re-answer gate therefore checks the owed escalation BEFORE the confirmed-delivery repair — dropping a promised human handoff is the worse failure. Neither branch reposts. Verification: turbo typecheck 10/10, turbo test 10/10 (1949 tests, up from 1924). Every fix carries red-green verification; 16 separate mutations were each observed RED and restored to GREEN, including one confirming that gating the MailboxHash path turns 10 tests red, which pins that boundary. Existing assertions that referenced the old markers were updated and each re-checked for degenerating to always-pass. Confirmed by diff that start.sh, both Dockerfiles, .github/workflows/ci.yml, .env.example and the 20260812 SystemConfig migration remain byte-identical to main. --- .../src/__tests__/postmark-webhook.test.ts | 376 +++++++++++++++++- .../src/app/api/webhooks/postmark/route.ts | 55 ++- .../src/app/api/webhooks/postmark/utils.ts | 91 ++++- .../migration.sql | 10 + packages/outpost/db/prisma/schema.prisma | 14 +- .../queue/src/__tests__/ai-response.test.ts | 316 ++++++++++++++- .../outpost/queue/src/handlers/ai-response.ts | 137 +++++-- 7 files changed, 927 insertions(+), 72 deletions(-) create mode 100644 packages/outpost/db/prisma/migrations/20260813000000_add_message_response_substates/migration.sql diff --git a/apps/web/src/__tests__/postmark-webhook.test.ts b/apps/web/src/__tests__/postmark-webhook.test.ts index aa90bc23..617b222c 100644 --- a/apps/web/src/__tests__/postmark-webhook.test.ts +++ b/apps/web/src/__tests__/postmark-webhook.test.ts @@ -7,10 +7,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; const mockTicketFindUnique = vi.fn(); const mockTicketFindFirst = vi.fn(); +const mockTicketFindMany = vi.fn(); const mockTicketCreate = vi.fn(); const mockTicketUpdate = vi.fn(); const mockMessageCreate = vi.fn(); -const mockMessageFindFirst = vi.fn(); +const mockMessageFindMany = vi.fn(); const mockJobCreate = vi.fn(); const mockTransaction = vi.fn(); @@ -20,12 +21,13 @@ vi.mock('@copilotkit/outpost/db', () => ({ ticket: { findUnique: (...args: unknown[]) => mockTicketFindUnique(...args), findFirst: (...args: unknown[]) => mockTicketFindFirst(...args), + findMany: (...args: unknown[]) => mockTicketFindMany(...args), create: (...args: unknown[]) => mockTicketCreate(...args), update: (...args: unknown[]) => mockTicketUpdate(...args), }, message: { create: (...args: unknown[]) => mockMessageCreate(...args), - findFirst: (...args: unknown[]) => mockMessageFindFirst(...args), + findMany: (...args: unknown[]) => mockMessageFindMany(...args), }, job: { create: (...args: unknown[]) => mockJobCreate(...args), @@ -62,7 +64,10 @@ import { extractReplyMessageIds, getHeaderValue, hasReplyHeaders, + isTicketParticipant, + MAX_REPLY_MESSAGE_IDS, normalizeMessageId, + normalizeParticipantEmail, } from '@/app/api/webhooks/postmark/utils'; import type { PostmarkInboundPayload } from '@/app/api/webhooks/postmark/utils'; @@ -99,10 +104,12 @@ describe('Postmark inbound webhook', () => { mockJobCreate.mockReset(); mockTransaction.mockReset(); mockCreateJob.mockReset(); - mockMessageFindFirst.mockReset(); + mockTicketFindMany.mockReset(); + mockMessageFindMany.mockReset(); mockTicketFindUnique.mockReset(); mockTicketFindFirst.mockResolvedValue(null); - mockMessageFindFirst.mockResolvedValue(null); + mockTicketFindMany.mockResolvedValue([]); + mockMessageFindMany.mockResolvedValue([]); mockTicketFindUnique.mockResolvedValue(null); mockJobCreate.mockResolvedValue({ id: 'job-1' }); mockCreateJob.mockResolvedValue('job-1'); @@ -230,6 +237,20 @@ describe('Postmark inbound webhook', () => { it('drops unparseable tokens', () => { expect(extractReplyMessageIds([{ Name: 'In-Reply-To', Value: '<>' }])).toEqual([]); }); + + it('caps a sender-supplied chain, keeping the oldest IDs that hold the thread root', () => { + const chain = Array.from({ length: 500 }, (_, i) => ``).join(' '); + const ids = extractReplyMessageIds([ + { Name: 'In-Reply-To', Value: '' }, + { Name: 'References', Value: chain }, + ]); + + expect(ids).toHaveLength(MAX_REPLY_MESSAGE_IDS); + // In-Reply-To first, then References oldest → newest, so the thread + // root survives truncation. + expect(ids[0]).toBe('newest@x'); + expect(ids[1]).toBe('id-0@x'); + }); }); describe('hasReplyHeaders', () => { @@ -251,6 +272,97 @@ describe('Postmark inbound webhook', () => { }); }); + describe('normalizeParticipantEmail', () => { + it('parses an addressed author label, case-folded', () => { + expect(normalizeParticipantEmail('Alice Smith ')).toBe( + 'alice@example.com', + ); + expect(normalizeParticipantEmail('alice@example.com')).toBe('alice@example.com'); + }); + + it('rejects non-address author labels other channels write to the same column', () => { + // Message.author is shared with every other source; none of these may + // ever be usable as a participant identity. + expect(normalizeParticipantEmail('Outpost AI')).toBeNull(); + expect(normalizeParticipantEmail('System')).toBeNull(); + expect(normalizeParticipantEmail('slack:U123456')).toBeNull(); + expect(normalizeParticipantEmail('octocat (583231)')).toBeNull(); + expect(normalizeParticipantEmail('alice@localhost')).toBeNull(); + expect(normalizeParticipantEmail('')).toBeNull(); + expect(normalizeParticipantEmail(undefined)).toBeNull(); + expect(normalizeParticipantEmail(null)).toBeNull(); + }); + }); + + // The header reply path is authorization-gated because In-Reply-To / + // References are attacker-supplied and a Message-ID is *known* to everyone + // who was ever on the thread, CCs included. + describe('isTicketParticipant', () => { + it('accepts the address recorded on an existing message author', () => { + expect( + isTicketParticipant('alice@example.com', { + messages: [{ author: 'Alice Smith ' }], + }), + ).toBe(true); + }); + + it('accepts the ticket\'s linked user email, case-insensitively', () => { + expect( + isTicketParticipant('ALICE@example.com', { + user: { email: 'alice@Example.com' }, + messages: [], + }), + ).toBe(true); + }); + + it('accepts a second address at the ticket account domain so aliases are not locked out', () => { + expect( + isTicketParticipant('a.smith@acme.com', { + account: { domain: 'ACME.com' }, + messages: [{ author: 'Alice Smith ' }], + }), + ).toBe(true); + // Tolerate a domain stored with a leading @. + expect( + isTicketParticipant('a.smith@acme.com', { + account: { domain: '@acme.com' }, + }), + ).toBe(true); + }); + + it('rejects an outsider who merely knows a Message-ID from the thread', () => { + expect( + isTicketParticipant('cc-observer@evil.test', { + user: { email: 'alice@example.com' }, + account: { domain: 'example.com' }, + messages: [ + { author: 'Alice Smith ' }, + { author: 'Outpost AI' }, + ], + }), + ).toBe(false); + }); + + it('does not treat the derived domain of a participant address as a domain match', () => { + // Otherwise every gmail.com sender would be a participant on any + // ticket opened from a gmail.com address. + expect( + isTicketParticipant('attacker@gmail.com', { + user: { email: 'victim@gmail.com' }, + messages: [{ author: 'Victim ' }], + }), + ).toBe(false); + }); + + it('rejects a bot-looking sender and an empty participant set', () => { + expect(isTicketParticipant('Outpost AI', { messages: [{ author: 'Outpost AI' }] })).toBe( + false, + ); + expect(isTicketParticipant('alice@example.com', {})).toBe(false); + expect(isTicketParticipant('', { messages: [{ author: null }] })).toBe(false); + }); + }); + // ── Route handler tests ──────────────────────────────────────────────── describe('POST handler', () => { @@ -575,30 +687,55 @@ describe('Postmark inbound webhook', () => { return list; } + /** + * A resolvable ticket whose participant set already contains the default + * payload sender (`alice@example.com`, recorded as the author of the + * opening message). Header resolution is authorization-gated, so every + * legitimate-reply fixture has to look like a real conversation. + */ + function ticket( + fields: { id: string; displayId: string; status: string }, + participants: { + messages?: Array<{ author: string | null }>; + user?: { email: string | null } | null; + account?: { domain: string | null } | null; + } = {}, + ) { + return { + ...fields, + user: participants.user ?? null, + account: participants.account ?? null, + messages: participants.messages ?? [ + { author: 'Alice Smith ' }, + { author: 'Outpost AI' }, + ], + }; + } + /** * Answer the reply-resolution ticket lookup (`sourceId: { in: [...] }`) * from a map, while leaving the MessageID idempotency lookup - * (`sourceId: ''`) returning null. + * (`sourceId: ''`, a `findFirst`) returning null. */ function ticketsBySourceId(map: Record) { - mockTicketFindFirst.mockImplementation(async (args: unknown) => { + mockTicketFindMany.mockImplementation(async (args: unknown) => { const where = (args as { where?: { sourceId?: { in?: string[] } } }).where; const ids = where?.sourceId?.in; - if (!Array.isArray(ids)) return null; - for (const id of ids) { - if (map[id]) return map[id]; - } - return null; + if (!Array.isArray(ids)) return []; + // Insertion order into `map` stands in for `orderBy createdAt asc`. + return Object.entries(map) + .filter(([id]) => ids.includes(id)) + .map(([, value]) => value); }); } it('appends a reply whose In-Reply-To matches a ticket sourceId, with no AI job', async () => { ticketsBySourceId({ - 'root-msg@postmark.example': { + 'root-msg@postmark.example': ticket({ id: 'ticket-root', displayId: 'TKT-ROOT0001', status: 'OPEN', - }, + }), }); mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); @@ -643,11 +780,11 @@ describe('Postmark inbound webhook', () => { // so a reply to our own message names an ID no row holds. References // still carries the customer's opening Message-ID. ticketsBySourceId({ - 'root-msg@postmark.example': { + 'root-msg@postmark.example': ticket({ id: 'ticket-root', displayId: 'TKT-ROOT0001', status: 'WAITING_ON_CUSTOMER', - }, + }), }); mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); mockTicketUpdate.mockResolvedValue({}); @@ -680,9 +817,15 @@ describe('Postmark inbound webhook', () => { it('resolves a reply that matches a mid-thread Message.attachments.postmarkMessageId', async () => { ticketsBySourceId({}); - mockMessageFindFirst.mockResolvedValue({ - ticket: { id: 'ticket-mid', displayId: 'TKT-MID00001', status: 'OPEN' }, - }); + mockMessageFindMany.mockResolvedValue([ + { + ticket: ticket({ + id: 'ticket-mid', + displayId: 'TKT-MID00001', + status: 'OPEN', + }), + }, + ]); mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); const res = await POST( @@ -698,7 +841,7 @@ describe('Postmark inbound webhook', () => { status: 'message_appended', ticketId: 'TKT-MID00001', }); - expect(mockMessageFindFirst).toHaveBeenCalledWith( + expect(mockMessageFindMany).toHaveBeenCalledWith( expect.objectContaining({ where: expect.objectContaining({ OR: [ @@ -748,6 +891,193 @@ describe('Postmark inbound webhook', () => { expect(mockCreateJob).not.toHaveBeenCalled(); }); + // ── Header reply path is authorization-gated ──────────────────────── + // + // In-Reply-To / References come from the sender, and a Message-ID is + // KNOWN to every thread participant — including anyone ever CC'd. Header + // matching alone therefore let an outsider append to, and reopen, someone + // else's ticket. The sender must already be a participant. + + it('does NOT append a non-participant who names a valid ticket Message-ID, and preserves their mail instead', async () => { + ticketsBySourceId({ + 'root-msg@postmark.example': ticket( + { id: 'ticket-victim', displayId: 'TKT-VICTIM01', status: 'RESOLVED' }, + { + user: { email: 'alice@example.com' }, + account: { domain: 'example.com' }, + messages: [ + { author: 'Alice Smith ' }, + { author: 'Outpost AI' }, + ], + }, + ), + }); + mockTicketCreate.mockResolvedValue({ + id: 'ticket-outsider', + displayId: 'TKT-TESTID01', + }); + + const res = await POST( + postmarkRequest( + fullPayload({ + // A CC on the thread: holds the Message-ID, is not a participant. + From: 'Eve Observer ', + FromName: 'Eve Observer', + MessageID: 'outsider-msg@postmark.example', + Subject: 'Re: Need help with billing', + TextBody: 'Please wire the payment to this account instead.', + Headers: headers(''), + }), + ), + ); + + expect(res.status).toBe(200); + // Not appended to the victim's ticket... + expect(mockMessageCreate).not.toHaveBeenCalled(); + // ...and the dormant victim ticket is NOT reopened. + expect(mockTicketUpdate).not.toHaveBeenCalled(); + + // ...but the mail is not dropped either: it is filed as its own + // ticket down the orphaned-reply path so a human still sees it. + expect(await res.json()).toMatchObject({ status: 'ticket_created' }); + expect(mockTicketCreate).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + description: 'Please wire the payment to this account instead.', + sourceId: 'outsider-msg@postmark.example', + messages: expect.objectContaining({ + create: expect.objectContaining({ + content: 'Please wire the payment to this account instead.', + author: 'Eve Observer ', + }), + }), + }), + }), + ); + // Being a reply, it never earns an AI response. + expect(mockJobCreate).not.toHaveBeenCalled(); + expect(mockCreateJob).not.toHaveBeenCalled(); + }); + + it('does NOT append a non-participant who names a valid mid-thread Message-ID', async () => { + ticketsBySourceId({}); + mockMessageFindMany.mockResolvedValue([ + { + ticket: ticket( + { id: 'ticket-victim', displayId: 'TKT-VICTIM01', status: 'CLOSED' }, + { messages: [{ author: 'Alice Smith ' }] }, + ), + }, + ]); + mockTicketCreate.mockResolvedValue({ id: 'ticket-outsider', displayId: 'TKT-TESTID01' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + From: 'Eve Observer ', + MessageID: 'outsider-msg@postmark.example', + Headers: headers(''), + }), + ), + ); + + expect(await res.json()).toMatchObject({ status: 'ticket_created' }); + expect(mockMessageCreate).not.toHaveBeenCalled(); + expect(mockTicketUpdate).not.toHaveBeenCalled(); + expect(mockJobCreate).not.toHaveBeenCalled(); + }); + + it('appends a reply from a second address at the ticket account domain', async () => { + ticketsBySourceId({ + 'root-msg@postmark.example': ticket( + { id: 'ticket-root', displayId: 'TKT-ROOT0001', status: 'OPEN' }, + { + account: { domain: 'example.com' }, + messages: [{ author: 'Alice Smith ' }], + }, + ), + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + From: 'Bob Jones ', + MessageID: 'colleague-msg@postmark.example', + Headers: headers(''), + }), + ), + ); + + expect(await res.json()).toMatchObject({ + status: 'message_appended', + ticketId: 'TKT-ROOT0001', + }); + expect(mockTicketCreate).not.toHaveBeenCalled(); + }); + + it('lands on the sender\'s own ticket when the chain also names someone else\'s older ticket', async () => { + ticketsBySourceId({ + // Oldest first — a plain "take the oldest row" resolution would + // pick the victim's ticket and then refuse the whole reply. + 'victim-root@postmark.example': ticket( + { id: 'ticket-victim', displayId: 'TKT-VICTIM01', status: 'OPEN' }, + { messages: [{ author: 'Carol ' }] }, + ), + 'own-root@postmark.example': ticket({ + id: 'ticket-own', + displayId: 'TKT-OWN00001', + status: 'OPEN', + }), + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + + const res = await POST( + postmarkRequest( + fullPayload({ + MessageID: 'reply-msg@postmark.example', + Headers: headers( + '', + ' ', + ), + }), + ), + ); + + expect(await res.json()).toMatchObject({ + status: 'message_appended', + ticketId: 'TKT-OWN00001', + }); + }); + + // The MailboxHash path is a plus-addressed token WE mint and hand out, so + // it stays ungated — adding a participant check there would break the + // legitimate reply path that already works. + it('still appends a MailboxHash reply from an address with no prior participation', async () => { + mockTicketFindUnique.mockResolvedValue({ + id: 'hash-ticket', + displayId: 'TKT-HASH0001', + status: 'RESOLVED', + }); + mockMessageCreate.mockResolvedValue({ id: 'msg-appended' }); + mockTicketUpdate.mockResolvedValue({}); + + const res = await POST( + postmarkRequest( + fullPayload({ + From: 'Alias ', + MailboxHash: 'TKT-HASH0001', + }), + ), + ); + + expect(await res.json()).toMatchObject({ + status: 'message_appended', + ticketId: 'TKT-HASH0001', + }); + expect(mockTicketUpdate).toHaveBeenCalled(); + }); + it('still answers a genuinely new email that carries headers but no threading headers', async () => { mockTicketCreate.mockResolvedValue({ id: 'ticket-new', displayId: 'TKT-TESTID01' }); @@ -773,7 +1103,8 @@ describe('Postmark inbound webhook', () => { }), }); // No threading headers means no header lookups at all. - expect(mockMessageFindFirst).not.toHaveBeenCalled(); + expect(mockMessageFindMany).not.toHaveBeenCalled(); + expect(mockTicketFindMany).not.toHaveBeenCalled(); }); it('treats an empty References header as not-a-reply', async () => { @@ -803,7 +1134,8 @@ describe('Postmark inbound webhook', () => { expect(await res.json()).toMatchObject({ ticketId: 'TKT-HASH0001' }); expect(mockTicketFindFirst).not.toHaveBeenCalled(); - expect(mockMessageFindFirst).not.toHaveBeenCalled(); + expect(mockTicketFindMany).not.toHaveBeenCalled(); + expect(mockMessageFindMany).not.toHaveBeenCalled(); }); it('files an unresolvable MailboxHash reply without falling back to headers', async () => { @@ -822,7 +1154,7 @@ describe('Postmark inbound webhook', () => { ), ); - expect(mockMessageFindFirst).not.toHaveBeenCalled(); + expect(mockMessageFindMany).not.toHaveBeenCalled(); expect(mockJobCreate).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/app/api/webhooks/postmark/route.ts b/apps/web/src/app/api/webhooks/postmark/route.ts index 572cc65e..9930b0b5 100644 --- a/apps/web/src/app/api/webhooks/postmark/route.ts +++ b/apps/web/src/app/api/webhooks/postmark/route.ts @@ -12,7 +12,10 @@ * Replies are detected first by `MailboxHash` (an exact ticket reference) and * then by the RFC 5322 threading headers `In-Reply-To` / `References`, which is * the only signal available when the customer's mail client replies to a plain - * From address and drops the plus-address. + * From address and drops the plus-address. Those headers are attacker-supplied, + * so that second path additionally requires the sender to already be a + * participant on the ticket it resolves to; `MailboxHash` is a token we mint and + * needs no such check. */ import crypto from 'node:crypto'; import { NextResponse } from 'next/server'; @@ -29,12 +32,26 @@ import { extractName, extractReplyMessageIds, hasReplyHeaders, + isTicketParticipant, } from './utils'; import type { PostmarkInboundPayload } from './utils'; /** Ticket fields the reply paths need. */ type ReplyTargetTicket = { id: string; displayId: string; status: string }; +/** + * Everything the header reply path needs: the append target plus the + * participant set it is authorized against. + */ +const REPLY_TARGET_SELECT = { + id: true, + displayId: true, + status: true, + user: { select: { email: true } }, + account: { select: { domain: true } }, + messages: { select: { author: true } }, +} as const; + /** * Resolve a reply to the ticket that already holds its conversation. * @@ -43,23 +60,39 @@ type ReplyTargetTicket = { id: string; displayId: string; status: string }; * appended message). A reply can name either, so both are checked. Outbound * Message-IDs are never persisted, which is why `References` (the full chain, * including the customer's own opening ID) matters as much as `In-Reply-To`. + * + * `In-Reply-To` / `References` are supplied by the sender, and a Message-ID is + * *known* to everyone who was ever on the thread — including a CC. Naming a + * valid ID is therefore not evidence of belonging to the ticket, so every + * candidate must additionally pass `isTicketParticipant(senderEmail, …)`. A + * candidate that resolves but fails that check is treated as unresolved, which + * sends the mail down the orphaned-reply path: filed as its own ticket for a + * human, never answered, never dropped. + * + * Candidates are scanned oldest-first rather than taking the single oldest row, + * so a chain naming both someone else's ticket and the sender's own still lands + * on the sender's own. */ async function findTicketByReplyMessageIds( messageIds: string[], + senderEmail: string, ): Promise { if (messageIds.length === 0) return null; // The opening email of a thread — the oldest match wins so a thread always // resolves to its root ticket. - const openingTicket = await prisma.ticket.findFirst({ + const openingTickets = await prisma.ticket.findMany({ where: { source: 'EMAIL', sourceId: { in: messageIds } }, orderBy: { createdAt: 'asc' }, - select: { id: true, displayId: true, status: true }, + select: REPLY_TARGET_SELECT, }); + const openingTicket = openingTickets.find((ticket) => + isTicketParticipant(senderEmail, ticket), + ); if (openingTicket) return openingTicket; // A message appended mid-thread. - const appendedMessage = await prisma.message.findFirst({ + const appendedMessages = await prisma.message.findMany({ where: { ticket: { source: 'EMAIL' }, OR: messageIds.map((id) => ({ @@ -67,9 +100,13 @@ async function findTicketByReplyMessageIds( })), }, orderBy: { createdAt: 'asc' }, - select: { ticket: { select: { id: true, displayId: true, status: true } } }, + select: { ticket: { select: REPLY_TARGET_SELECT } }, }); - return appendedMessage?.ticket ?? null; + return ( + appendedMessages + .map((message) => message?.ticket) + .find((ticket) => ticket && isTicketParticipant(senderEmail, ticket)) ?? null + ); } function isUniqueConstraintError(error: unknown): boolean { @@ -199,9 +236,13 @@ export async function POST(request: Request) { } else { // Fallback: most mail clients reply to the plain From address and // never preserve the plus-address, so RFC 5322 threading headers are - // the only thing marking those as replies. + // the only thing marking those as replies. Unlike MailboxHash — a + // plus-addressed token WE mint and hand out — these headers are + // attacker-supplied, so the sender must already be a participant on + // the ticket they name. const replyTarget = await findTicketByReplyMessageIds( extractReplyMessageIds(body.Headers), + senderEmail, ); if (replyTarget) { return await appendReplyToTicket(replyTarget, body, author, messageBody); diff --git a/apps/web/src/app/api/webhooks/postmark/utils.ts b/apps/web/src/app/api/webhooks/postmark/utils.ts index ce069bf8..9d44b864 100644 --- a/apps/web/src/app/api/webhooks/postmark/utils.ts +++ b/apps/web/src/app/api/webhooks/postmark/utils.ts @@ -60,6 +60,18 @@ export function hasReplyHeaders( return REPLY_HEADERS.some((name) => (getHeaderValue(headers, name) ?? '').trim().length > 0); } +/** + * Cap on candidate Message-IDs taken from one payload. + * + * The header value is sender-supplied and unbounded, and each candidate widens + * an `IN (…)` lookup that now returns every match with its participant set. + * Truncating is safe for resolution: `In-Reply-To` is collected first and + * `References` runs oldest → newest, so the thread root — the ID stored as + * `Ticket.sourceId` — is always near the front. Real chains are far under this; + * RFC 5322 §3.6.4 already expects clients to trim long ones. + */ +export const MAX_REPLY_MESSAGE_IDS = 50; + /** * Collect every candidate Message-ID a reply points at, normalized and deduped. * @@ -69,7 +81,8 @@ export function hasReplyHeaders( * opening Message-ID — the value stored as `Ticket.sourceId`. * * Order is In-Reply-To first, then the References chain as sent (oldest → - * newest). Callers match the whole set at once, so order is informational. + * newest). Callers match the whole set at once, so order is informational except + * where `MAX_REPLY_MESSAGE_IDS` truncates. */ export function extractReplyMessageIds( headers: Array<{ Name: string; Value: string }> | undefined, @@ -82,11 +95,87 @@ export function extractReplyMessageIds( for (const token of value.split(/[\s,]+/)) { const id = normalizeMessageId(token); if (id && !ids.includes(id)) ids.push(id); + if (ids.length >= MAX_REPLY_MESSAGE_IDS) return ids; } } return ids; } +/** + * Parse a stored participant label down to a comparable email address. + * + * `Message.author` holds free-form labels from every channel — `"Alice Smith + * "` from this webhook, but also `"Outpost AI"`, `"System"`, + * `"slack:U123"` and `"octocat (583231)"`. Only values that actually parse to an + * address may be treated as a participant, otherwise a sender literally named + * `System` would inherit every ticket the escalation handler ever touched. + */ +export function normalizeParticipantEmail(raw: string | undefined | null): string | null { + if (!raw) return null; + const candidate = extractEmail(raw).trim().toLowerCase(); + return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(candidate) ? candidate : null; +} + +/** Domain part of an already-normalized address. */ +export function emailDomain(email: string | null | undefined): string | null { + if (!email) return null; + const at = email.lastIndexOf('@'); + return at > 0 && at < email.length - 1 ? email.slice(at + 1) : null; +} + +/** The participant-bearing fields the header reply path reads off a ticket. */ +export interface TicketParticipants { + user?: { email: string | null } | null; + account?: { domain: string | null } | null; + messages?: Array<{ author: string | null } | null> | null; +} + +/** + * True when `senderEmail` is already part of this ticket's conversation. + * + * `In-Reply-To` / `References` are attacker-controlled: a Message-ID is *known* + * to every thread participant (including anyone ever CC'd), so header matching + * alone lets an outsider append to — and reopen — someone else's ticket. This is + * the authorization half of that lookup. + * + * A participant is: + * 1. the ticket's linked `user.email`; + * 2. any address parsed out of an existing `Message.author` on the ticket — + * this is the load-bearing one, since it covers whoever opened the thread + * plus anyone (customer or team member) who has already replied by email; + * 3. anybody at the ticket's `account.domain`, so a colleague or a second + * address on the same thread is not locked out. + * + * Deliberately NOT a participant: someone sharing the *derived* domain of + * `user.email` or of a message author. Inferring the domain from a participant's + * address would make every `gmail.com` sender a participant on any ticket opened + * from a `gmail.com` address — most consumer tickets — which reintroduces the + * hole. `Account.domain` is company data a human deliberately set on the CRM + * record, so it cannot silently widen to a public mail provider. + * + * Aliases that match none of the three are not dropped: the caller falls through + * to the orphaned-reply path, which files the message as its own ticket for a + * human and never spends an AI response on it. + */ +export function isTicketParticipant( + senderEmail: string | undefined | null, + ticket: TicketParticipants, +): boolean { + const sender = normalizeParticipantEmail(senderEmail); + if (!sender) return false; + + if (normalizeParticipantEmail(ticket.user?.email) === sender) return true; + + for (const message of ticket.messages ?? []) { + if (normalizeParticipantEmail(message?.author) === sender) return true; + } + + const accountDomain = ticket.account?.domain?.trim().toLowerCase().replace(/^@/, ''); + if (accountDomain && emailDomain(sender) === accountDomain) return true; + + return false; +} + /** Postmark inbound webhook payload (relevant fields). */ export interface PostmarkInboundPayload { From: string; diff --git a/packages/outpost/db/prisma/migrations/20260813000000_add_message_response_substates/migration.sql b/packages/outpost/db/prisma/migrations/20260813000000_add_message_response_substates/migration.sql new file mode 100644 index 00000000..b2b2e0ab --- /dev/null +++ b/packages/outpost/db/prisma/migrations/20260813000000_add_message_response_substates/migration.sql @@ -0,0 +1,10 @@ +-- Move the two PENDING sub-states of a primary AI response out of the free-text +-- responseError column, which an operations surface reads as "what went wrong". +-- +-- Purely additive: both columns are new, deliveryConfirmed carries a DEFAULT so +-- existing rows are correct without a backfill (no historical row was ever +-- delivery-confirmed), and escalationRequiredReason is NULL for every existing +-- row, which is exactly "no escalation is owed". No unique index is created. +ALTER TABLE "Message" +ADD COLUMN "deliveryConfirmed" BOOLEAN NOT NULL DEFAULT false, +ADD COLUMN "escalationRequiredReason" TEXT; diff --git a/packages/outpost/db/prisma/schema.prisma b/packages/outpost/db/prisma/schema.prisma index 69ee89be..824649bc 100644 --- a/packages/outpost/db/prisma/schema.prisma +++ b/packages/outpost/db/prisma/schema.prisma @@ -111,7 +111,19 @@ model Message { responseKey String? // Idempotency slot; PRIMARY_AI_RESPONSE is unique per ticket responseState MessageResponseState? // Lifecycle of the primary AI response responseJobId String? // Queue job that owns a PENDING primary response - responseError String? // Last delivery failure, used by retry recovery + responseError String? // Last real delivery/bookkeeping error text — never lifecycle state + // Two sub-states of a PENDING primary response, kept OUT of responseError so + // that column stays readable as "what went wrong" on an ops surface. + // deliveryConfirmed — the platform post succeeded but the + // PENDING -> DELIVERED write did not, so a retry + // must repair state instead of reposting. + // escalationRequiredReason — non-null means a human handoff is owed and not + // yet durable; it holds the reason to enqueue. + // Neither is a MessageResponseState value: responseState records the OUTCOME + // (PENDING/DELIVERED/ESCALATED) and both of these describe a response that is + // still PENDING, so folding them into the enum would lose the outcome. + deliveryConfirmed Boolean @default(false) + escalationRequiredReason String? attachments Json? createdAt DateTime @default(now()) diff --git a/packages/outpost/queue/src/__tests__/ai-response.test.ts b/packages/outpost/queue/src/__tests__/ai-response.test.ts index 89f16ede..aa5d733e 100644 --- a/packages/outpost/queue/src/__tests__/ai-response.test.ts +++ b/packages/outpost/queue/src/__tests__/ai-response.test.ts @@ -19,6 +19,7 @@ const mockPrismaMessage = { create: vi.fn(), update: vi.fn(), updateMany: vi.fn(), + findUnique: vi.fn(), }; const mockPrismaJob = { @@ -249,6 +250,9 @@ describe('handleAiResponse', () => { mockPrismaMessage.create.mockResolvedValue({ id: 'msg-new' }); mockPrismaMessage.update.mockResolvedValue({}); mockPrismaMessage.updateMany.mockResolvedValue({ count: 1 }); + // Only read when an escalation compare-and-set reports no rows changed; + // "row is gone" is the least forgiving default for that path. + mockPrismaMessage.findUnique.mockResolvedValue(null); mockPrismaJob.create.mockResolvedValue({ id: 'job-esc-1' }); mockPrismaQueryRaw.mockResolvedValue([{ now: new Date('2026-08-11T20:00:00.000Z') }]); mockPrismaTransaction.mockImplementation( @@ -936,7 +940,8 @@ describe('handleAiResponse', () => { expect(mockPrismaMessage.update).toHaveBeenCalledWith({ where: { id: 'msg-new' }, data: { - responseError: expect.stringContaining('DELIVERY_CONFIRMED'), + deliveryConfirmed: true, + responseError: expect.stringContaining('DB write conflict'), }, }); }); @@ -954,7 +959,9 @@ describe('handleAiResponse', () => { responseKey: 'PRIMARY_AI_RESPONSE', responseState: 'PENDING', responseJobId: 'job-delivered-state', - responseError: 'DELIVERY_CONFIRMED: DB write conflict', + deliveryConfirmed: true, + responseError: + 'Delivery succeeded but the DELIVERED state write failed: DB write conflict', createdAt: new Date(), }, ], @@ -999,6 +1006,111 @@ describe('handleAiResponse', () => { expect(context.reportProgress).toHaveBeenLastCalledWith(85); }); + /** + * The escalation compare-and-set reports "no rows changed" instead of + * throwing when the response row is no longer PENDING. Nothing was + * queued in that case, so what the handler reports has to follow the + * row's actual state — not the local conditions that asked for the + * escalation. `count: 0` on updateMany is exactly that no-op. + */ + describe('escalation compare-and-set changed no rows', () => { + beforeEach(() => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(lowConfidenceResult); + mockPrismaMessage.updateMany.mockResolvedValue({ count: 0 }); + }); + + it('fails loudly when the response is still PENDING, so no human was summoned', async () => { + mockPrismaMessage.findUnique.mockResolvedValue({ responseState: 'PENDING' }); + const context = makeContext(); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + + expect(result.success).toBe(false); + expect(result.error).toContain('was not queued'); + expect(result.error).toContain('PENDING'); + expect(result.error).toContain('Low AI confidence'); + // No ESCALATION row was written: the transaction rolled back. + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + // Same rule as the enqueue-threw case: a failing attempt must not + // look complete on the DEAD_LETTER row. + expect(context.reportProgress).not.toHaveBeenCalledWith(100); + }); + + it('fails loudly when the response row cannot be found', async () => { + mockPrismaMessage.findUnique.mockResolvedValue(null); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(false); + expect(result.error).toContain('response state is missing'); + }); + + it('fails loudly when the response state cannot be read', async () => { + mockPrismaMessage.findUnique.mockRejectedValue(new Error('db down')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(false); + expect(result.error).toContain('unreadable (db down)'); + }); + + it('succeeds with escalated:false when the response is already DELIVERED', async () => { + // No handoff was owed: the reporter has a durable answer, so a + // no-op enqueue is unremarkable and `escalated` must match the row. + mockPrismaMessage.findUnique.mockResolvedValue({ responseState: 'DELIVERED' }); + const context = makeContext(); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + context, + ); + + expect(result.success).toBe(true); + expect(result.data?.escalated).toBe(false); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(context.reportProgress).toHaveBeenCalledWith(100); + }); + + it('succeeds with escalated:true when another actor already escalated the row', async () => { + mockPrismaMessage.findUnique.mockResolvedValue({ responseState: 'ESCALATED' }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + // A human is on it — just not because of this attempt. + expect(result.data?.escalated).toBe(true); + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + }); + + it('still reports escalated:true when this attempt does queue the escalation', async () => { + // Guards the rewritten `escalated`: the ordinary success path must + // keep reporting true off the committed enqueue. + mockPrismaMessage.updateMany.mockResolvedValue({ count: 1 }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext(), + ); + + expect(result.success).toBe(true); + expect(result.data?.escalated).toBe(true); + expect(mockPrismaMessage.findUnique).not.toHaveBeenCalled(); + }); + }); + it('retries the escalation after delivery and escalation both fail', async () => { const pendingResponse = { id: 'msg-new', @@ -1151,9 +1263,7 @@ describe('handleAiResponse', () => { expect(mockPrismaMessage.update).toHaveBeenCalledWith({ where: { id: 'msg-new' }, data: { - responseError: expect.stringContaining( - `ESCALATION_REQUIRED: ${reasonFragment}`, - ), + escalationRequiredReason: expect.stringContaining(reasonFragment), }, }); expect(mockPrismaMessage.update).not.toHaveBeenCalledWith({ @@ -1176,8 +1286,7 @@ describe('handleAiResponse', () => { responseKey: 'PRIMARY_AI_RESPONSE', responseState: 'PENDING', responseJobId: 'job-required-escalation', - responseError: - 'ESCALATION_REQUIRED: Low AI confidence (25%) — automated escalation', + escalationRequiredReason: 'Low AI confidence (25%) — automated escalation', createdAt: new Date(), }, ], @@ -1226,7 +1335,11 @@ describe('handleAiResponse', () => { responseKey: 'PRIMARY_AI_RESPONSE', responseState: 'PENDING', }, - data: { responseState: 'ESCALATED', responseError: null }, + data: { + responseState: 'ESCALATED', + responseError: null, + escalationRequiredReason: null, + }, }); }); @@ -1252,8 +1365,7 @@ describe('handleAiResponse', () => { responseKey: 'PRIMARY_AI_RESPONSE', responseState: 'PENDING', responseJobId: 'job-required-escalation', - responseError: - 'ESCALATION_REQUIRED: Low AI confidence (25%) — automated escalation', + escalationRequiredReason: 'Low AI confidence (25%) — automated escalation', createdAt: new Date('2026-04-23T10:00:20Z'), }, ], @@ -1275,9 +1387,191 @@ describe('handleAiResponse', () => { responseKey: 'PRIMARY_AI_RESPONSE', responseState: 'PENDING', }, - data: { responseState: 'ESCALATED', responseError: null }, + data: { + responseState: 'ESCALATED', + responseError: null, + escalationRequiredReason: null, + }, + }); + expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + }); + + // Lifecycle state is carried by dedicated columns, never by a prefix inside + // responseError. + // + // responseError is the free-text "what went wrong" column an operations + // surface renders verbatim, so a `DELIVERY_CONFIRMED:` value there showed the + // opposite of its meaning — a delivered answer displayed as a failure. The two + // signals are sub-states of a PENDING primary response, not outcomes, so they + // are not MessageResponseState values either: `deliveryConfirmed` is a + // boolean, and `escalationRequiredReason` is one nullable column holding both + // the fact that a handoff is owed and the reason to enqueue, so the flag and + // its payload cannot drift apart. + describe('lifecycle state stays out of responseError', () => { + /** Every responseError value this run wrote, ignoring explicit clears. */ + function writtenResponseErrors(): string[] { + return mockPrismaMessage.update.mock.calls + .map((call: Array<{ data: Record }>) => call[0].data.responseError) + .filter((value: unknown): value is string => typeof value === 'string'); + } + + it('records a confirmed delivery on its own column and keeps responseError as error text', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockPrismaMessage.update.mockImplementation( + async (args: { data: Record }) => { + if (args.data.responseState === 'DELIVERED') { + throw new Error('DB write conflict'); + } + return {}; + }, + ); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-confirmed-column' }), + ); + + expect(result.success).toBe(true); + const confirmations = mockPrismaMessage.update.mock.calls.filter( + (call: Array<{ data: Record }>) => + call[0].data.deliveryConfirmed === true, + ); + expect(confirmations).toHaveLength(1); + // The only thing responseError may carry is the write failure itself. + const errors = writtenResponseErrors(); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('DB write conflict'); + // No value written to the free-text column may be a state token. + for (const error of errors) { + expect(error).not.toMatch(/^[A-Z][A-Z_]+:/); + } + }); + + it('records an owed escalation on its own column and never in responseError', async () => { + mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket); + mockGenerateSupportResponse.mockResolvedValue(lowConfidenceResult); + mockPrismaJob.create.mockRejectedValue(new Error('queue unavailable')); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-owed-column' }), + ); + + expect(result.success).toBe(false); + expect(mockPrismaMessage.update).toHaveBeenCalledWith({ + where: { id: 'msg-new' }, + data: { + escalationRequiredReason: 'Low AI confidence (25%) — automated escalation', + }, }); + // The reason is not an error, so it must not reach responseError — + // delivery succeeded here, only the handoff is outstanding. + expect(writtenResponseErrors()).toEqual([]); + }); + + it.each([ + ['DELIVERY_CONFIRMED: fabricated confirmation'], + ['ESCALATION_REQUIRED: fabricated reason'], + ])('ignores responseError %j when the state columns are unset', async (spoofedError) => { + // An upstream platform error message is free text this handler + // does not author, so it must not be able to drive the lifecycle. + // These rows carry text in the exact old marker shape, prefix + // included, while both state columns say otherwise. + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-spoofed-markers', + type: 'BOT', + content: highConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-someone-else', + deliveryConfirmed: false, + escalationRequiredReason: null, + responseError: spoofedError, + createdAt: new Date(), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-spoofed-markers' }), + ); + + expect(result.data).toMatchObject({ skipped: true, reason: 'already_answered' }); + // Neither an escalation from the fake ESCALATION_REQUIRED text... + expect(mockPrismaJob.create).not.toHaveBeenCalled(); + expect(mockPrismaMessage.updateMany).not.toHaveBeenCalled(); + // ...nor a DELIVERED repair from the fake DELIVERY_CONFIRMED text. + expect(mockPrismaMessage.update).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); expect(mockGenerateSupportResponse).not.toHaveBeenCalled(); + }); + + it('recovers an owed escalation whose responseError holds an unrelated real error', async () => { + // Separate columns mean both can be set at once, which the single + // prefix could never represent. The owed handoff must still win: its + // transition is what ends the PENDING state, and no branch here + // reposts to the reporter. + mockPrismaTicket.findUnique.mockResolvedValue({ + ...sampleTicket, + messages: [ + ...sampleTicket.messages, + { + id: 'msg-owed-with-error', + type: 'BOT', + content: lowConfidenceResult.response, + isAiGenerated: true, + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + responseJobId: 'job-owed-with-error', + deliveryConfirmed: true, + escalationRequiredReason: 'Low AI confidence (25%) — automated escalation', + responseError: + 'Delivery succeeded but the DELIVERED state write failed: DB write conflict', + createdAt: new Date(), + }, + ], + }); + + const result = await handleAiResponse( + { ticketId: 'tkt-1', source: 'discord' }, + makeContext({ jobId: 'job-owed-with-error' }), + ); + + expect(result.success).toBe(true); + expect(result.data).toMatchObject({ + skipped: true, + escalated: true, + reason: 'escalation_recovered', + }); + expect(mockPrismaJob.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ type: 'ESCALATION' }), + }); + expect(mockPrismaJob.create.mock.calls[0][0].data.payload).toEqual({ + ticketId: 'tkt-1', + reason: 'Low AI confidence (25%) — automated escalation', + }); + // The ESCALATED transition clears both the stale error text and the + // owed-handoff marker in the same compare-and-set. + expect(mockPrismaMessage.updateMany).toHaveBeenCalledWith({ + where: { + id: 'msg-owed-with-error', + responseKey: 'PRIMARY_AI_RESPONSE', + responseState: 'PENDING', + }, + data: { + responseState: 'ESCALATED', + responseError: null, + escalationRequiredReason: null, + }, + }); expect(mockPostResponse).not.toHaveBeenCalled(); }); }); diff --git a/packages/outpost/queue/src/handlers/ai-response.ts b/packages/outpost/queue/src/handlers/ai-response.ts index d617710e..c1b1ddba 100644 --- a/packages/outpost/queue/src/handlers/ai-response.ts +++ b/packages/outpost/queue/src/handlers/ai-response.ts @@ -41,8 +41,6 @@ import type { AiResponsePayload, JobResult, JobHandlerContext } from '../types.j const PRIMARY_AI_RESPONSE_KEY = 'PRIMARY_AI_RESPONSE'; const RESPONSE_RECOVERY_AFTER_MS = 5 * 60 * 1000; -const DELIVERY_CONFIRMED_MARKER = 'DELIVERY_CONFIRMED'; -const ESCALATION_REQUIRED_MARKER = 'ESCALATION_REQUIRED'; interface StoredAiResponse { id: string; @@ -52,6 +50,8 @@ interface StoredAiResponse { responseState?: string | null; responseJobId?: string | null; responseError?: string | null; + deliveryConfirmed?: boolean | null; + escalationRequiredReason?: string | null; } /** @@ -81,8 +81,14 @@ function isPrimaryAiResponseConflict(error: unknown): boolean { ); } +/** + * True when the platform post is a proven fact even though responseState never + * made it out of PENDING. A dedicated flag rather than a prefix in + * responseError: that column is read as error text, and "the reporter has their + * answer" is the opposite of an error. + */ function hasConfirmedDelivery(response: StoredAiResponse): boolean { - return response.responseError?.startsWith(`${DELIVERY_CONFIRMED_MARKER}:`) ?? false; + return response.deliveryConfirmed === true; } /** @@ -105,7 +111,13 @@ async function enqueueEscalationAtomically( responseKey: PRIMARY_AI_RESPONSE_KEY, responseState: 'PENDING', }, - data: { responseState: 'ESCALATED', responseError: null }, + // The handoff is durable as of this transaction, so the "owed" + // marker is cleared with it. + data: { + responseState: 'ESCALATED', + responseError: null, + escalationRequiredReason: null, + }, }); if (transition.count !== 1) return false; @@ -123,16 +135,23 @@ async function enqueueEscalationAtomically( }); } +/** + * The human handoff this response promised but has not yet made durable. + * + * One nullable column carries both the fact and its payload, so the flag and the + * reason cannot drift apart: non-null means "escalation owed", and the value is + * the reason to enqueue. It is deliberately not a MessageResponseState value — + * the response is still PENDING, which is the outcome the enum records. + */ function requiredEscalationReason(response: StoredAiResponse): string | null { - const prefix = `${ESCALATION_REQUIRED_MARKER}: `; if ( response.responseKey !== PRIMARY_AI_RESPONSE_KEY || response.responseState !== 'PENDING' || - !response.responseError?.startsWith(prefix) + !response.escalationRequiredReason ) { return null; } - return response.responseError.slice(prefix.length); + return response.escalationRequiredReason; } async function recoverRequiredEscalation( @@ -362,6 +381,21 @@ export async function handleAiResponse( generatedResponses.find((m) => m.responseKey === PRIMARY_AI_RESPONSE_KEY) ?? generatedResponses[0]; if (priorAiResponse) { + // Order matters. The two PENDING sub-states now live in independent + // columns, so nothing at the type level stops a row carrying both. An + // owed human handoff is checked first because dropping it is the worse + // failure: its transition also ends the PENDING state, and neither branch + // ever reposts to the reporter. + const escalationRetryReason = requiredEscalationReason(priorAiResponse); + if (escalationRetryReason) { + return recoverRequiredEscalation( + ticketId, + priorAiResponse, + escalationRetryReason, + context, + ); + } + if (priorAiResponse.responseState === 'PENDING' && hasConfirmedDelivery(priorAiResponse)) { // The platform post succeeded; only the state mirror failed. Repair // it when possible, but never route an already-answered reporter to @@ -384,16 +418,6 @@ export async function handleAiResponse( }; } - const escalationRetryReason = requiredEscalationReason(priorAiResponse); - if (escalationRetryReason) { - return recoverRequiredEscalation( - ticketId, - priorAiResponse, - escalationRetryReason, - context, - ); - } - if ( priorAiResponse.responseKey === PRIMARY_AI_RESPONSE_KEY && priorAiResponse.responseState === 'PENDING' && @@ -498,6 +522,14 @@ export async function handleAiResponse( // report success until that handoff is durable. let escalationEnqueueError: string | null = null; let escalationReason: string | null = null; + // Whether this attempt's ESCALATION actually committed, and — when the + // compare-and-set found the response row already outside PENDING — which + // state it settled in. Read after the pipeline is torn down so the returned + // `escalated` reports what happened to the row instead of restating the + // local conditions that asked for an escalation. + let escalationEnqueued = false; + let escalationSkippedState: string | null = null; + let escalationStateReadError: string | null = null; let pipelineResult; try { @@ -728,9 +760,7 @@ export async function handleAiResponse( try { await prisma.message.update({ where: { id: aiMessage.id }, - data: { - responseError: `${ESCALATION_REQUIRED_MARKER}: ${nonDeliveryEscalationReason}`, - }, + data: { escalationRequiredReason: nonDeliveryEscalationReason }, }); } catch (error) { console.error( @@ -750,19 +780,21 @@ export async function handleAiResponse( `[AI Response] Failed to record durable delivery for ticket ${ticketId}:`, message, ); - // Delivery is already a fact. Persist a separate confirmation - // marker so a retry can repair the state without reposting or - // escalating an already-answered reporter. + // Delivery is already a fact. Persist it on its own flag so a + // retry can repair the state without reposting or escalating + // an already-answered reporter. The write failure itself is a + // genuine error, so it — and only it — goes in responseError. try { await prisma.message.update({ where: { id: aiMessage.id }, data: { - responseError: `${DELIVERY_CONFIRMED_MARKER}: ${message}`, + deliveryConfirmed: true, + responseError: `Delivery succeeded but the DELIVERED state write failed: ${message}`, }, }); } catch (markerError) { console.error( - `[AI Response] Failed to record delivery confirmation marker for ticket ${ticketId}:`, + `[AI Response] Failed to record delivery confirmation for ticket ${ticketId}:`, markerError instanceof Error ? markerError.message : String(markerError), @@ -799,7 +831,6 @@ export async function handleAiResponse( ? `AI response generated but not delivered to ${ticket.source} (${deliveryFailure}) — needs a human to answer the reporter` : nonDeliveryEscalationReason; - let escalationEnqueued = false; if (escalationReason) { try { escalationEnqueued = await enqueueEscalationAtomically( @@ -814,6 +845,32 @@ export async function handleAiResponse( escalationEnqueueError, ); } + + if (!escalationEnqueued && !escalationEnqueueError) { + // The compare-and-set found the row outside PENDING, so nothing + // was queued. Read the state it settled in before deciding what + // to report: DELIVERED means the answer is durable and no + // handoff was owed, ESCALATED means another actor already + // summoned the human. Anything else leaves the promised handoff + // unaccounted for and must not be reported as handled. Both + // terminal states are final in this handler, so reading them + // after the transaction cannot observe a third value. + try { + const settled = await prisma.message.findUnique({ + where: { id: aiMessage.id }, + select: { responseState: true }, + }); + escalationSkippedState = settled?.responseState ?? null; + } catch (error) { + escalationStateReadError = + error instanceof Error ? error.message : String(error); + } + console.warn( + `[AI Response] Ticket ${ticketId}: escalation (${escalationReason}) was not ` + + `queued — response row state is ${escalationSkippedState ?? 'unavailable'}` + + `${escalationStateReadError ? ` (${escalationStateReadError})` : ''}`, + ); + } } } finally { pipeline.destroy(); @@ -826,10 +883,12 @@ export async function handleAiResponse( `${deliveryFailure ? `, delivery failed (${deliveryFailure})` : ''}`, ); - const escalated = - deliveryFailure !== null || - pipelineResult.suppressed || - pipelineResult.confidenceScore < AI_CONFIDENCE.ESCALATE; + // A human was summoned only if this attempt's escalation committed, or if the + // row shows another actor already committed one. `escalationReason !== null` + // is exactly the old local-condition test (delivery failure, suppression, or + // sub-threshold confidence); what is new is that it no longer stands alone. + const escalationHandoffDurable = escalationEnqueued || escalationSkippedState === 'ESCALATED'; + const escalated = escalationReason !== null && escalationHandoffDurable; // A promised human handoff is part of successful completion even when the AI // response reached the reporter. Report enqueue failure so the queue retries: @@ -845,6 +904,24 @@ export async function handleAiResponse( }; } + // The enqueue reported no-op rather than throwing. DELIVERED is the one + // unremarkable explanation — the response is durably answered, so no handoff + // was owed and `escalated: false` matches the row. Every other state (still + // PENDING, row gone, or unreadable) means a reporter was promised a human who + // was never summoned: fail so the queue retries through the prior-response + // gate, which escalates without regenerating or reposting. + if (escalationReason && !escalationHandoffDurable && escalationSkippedState !== 'DELIVERED') { + const observed = escalationStateReadError + ? `unreadable (${escalationStateReadError})` + : (escalationSkippedState ?? 'missing'); + return { + success: false, + error: + `Ticket ${ticketId}: required escalation (${escalationReason}) was not queued — ` + + `response state is ${observed} — needs manual attention`, + }; + } + await context.reportProgress(100); return {