diff --git a/packages/outpost/ai/src/eval/harness.test.ts b/packages/outpost/ai/src/eval/harness.test.ts new file mode 100644 index 0000000..32d45ec --- /dev/null +++ b/packages/outpost/ai/src/eval/harness.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from 'vitest'; +import { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './harness.js'; +import { RULES } from './rules.js'; + +describe('scoreCases', () => { + it('reports every rule for every case', () => { + const report = scoreCases(HISTORICAL_FAILURES); + expect(report.totalCases).toBe(HISTORICAL_FAILURES.length); + for (const rule of RULES) { + expect(report.perRule[rule].total).toBe(HISTORICAL_FAILURES.length); + } + }); + + it('counts a clean case as clean', () => { + const report = scoreCases([TARGET_SHAPE]); + expect(report.cleanCases).toBe(1); + expect(report.cases[0].failed).toEqual([]); + }); +}); + +// The point of the golden set: every documented failure has to be caught by at +// least one rule. If a rule regresses, one of these stops failing — and a case +// that stops failing is exactly as alarming as a test that stops passing. +describe('the documented failures are all caught', () => { + it.each(HISTORICAL_FAILURES.map((c) => [c.id, c] as const))( + '%s breaks at least one rule', + (_id, testCase) => { + const report = scoreCases([testCase]); + expect(report.cases[0].failed.length).toBeGreaterThan(0); + }, + ); + + // Named individually rather than only in aggregate, so a regression says + // WHICH failure mode stopped being caught. + it('catches case A for hedging a name and for citing nothing', () => { + const failed = scoreCases([HISTORICAL_FAILURES[0]]).cases[0].failed; + expect(failed).toContain('no-hedged-names'); + expect(failed).toContain('no-banned-phrases'); + }); + + it('catches case B for the dead package and the invented name', () => { + const failed = scoreCases([HISTORICAL_FAILURES[1]]).cases[0].failed; + expect(failed).toContain('no-dead-package'); + expect(failed).toContain('grounded-identifiers'); + }); + + it('catches case C for the false capability claim', () => { + const failed = scoreCases([HISTORICAL_FAILURES[2]]).cases[0].failed; + expect(failed).toContain('no-banned-phrases'); + }); + + it('catches case D for self-commentary and issue-writing advice', () => { + const failed = scoreCases([HISTORICAL_FAILURES[3]]).cases[0].failed; + expect(failed).toContain('no-banned-phrases'); + }); +}); + +// A rule set that fires on everything is as useless as one that never fires, and +// this is the direction that costs a reporter a correct answer. +describe('the target shape passes', () => { + it('does not flag the maintainer answer the doc holds up as correct', () => { + const report = scoreCases([TARGET_SHAPE]); + expect(report.cases[0].failed).toEqual([]); + }); +}); + +// A live run whose fixture loading silently produced nothing used to render as a +// perfect score: every rule {passed: 0, total: 0}, so `passed === total` printed +// six `ok` lines. +describe('an empty case list', () => { + it('is refused rather than scored as clean', () => { + expect(() => scoreCases([])).toThrow(/no cases/i); + }); +}); + +describe('formatReport', () => { + it('leads with the clean count and names each failing rule', () => { + const text = formatReport(scoreCases(HISTORICAL_FAILURES)); + expect(text).toContain(`0/${HISTORICAL_FAILURES.length} cases clean`); + expect(text).toContain('no-dead-package'); + expect(text).toContain('Failing cases:'); + }); + + it('says nothing about failing cases when there are none', () => { + const text = formatReport(scoreCases([TARGET_SHAPE])); + expect(text).toContain('1/1 cases clean'); + expect(text).not.toContain('Failing cases:'); + }); +}); diff --git a/packages/outpost/ai/src/eval/harness.ts b/packages/outpost/ai/src/eval/harness.ts new file mode 100644 index 0000000..6755625 --- /dev/null +++ b/packages/outpost/ai/src/eval/harness.ts @@ -0,0 +1,240 @@ +/** + * Scoring and aggregation for the response-quality eval. + * + * `rules.ts` judges one reply. This turns a set of judged replies into the + * numbers the doc asks for — a per-rule pass rate, and the list of cases that + * regressed — so "did answer quality move?" has an answer that isn't someone's + * impression of the last few threads they read. + * + * ## How this gets driven + * + * `scoreCases` takes replies that were *already produced*. It deliberately does + * not call the pipeline itself, because the two ways of producing a reply have + * opposite requirements and only one belongs in CI: + * + * - **Fixed replies** (this file's `HISTORICAL_FAILURES`) — deterministic, no + * network, no model. Pins the rule set against known-bad output so a rule + * cannot silently stop firing. Runs in CI. + * - **Live replies** — real Pathfinder retrieval and a real model call, run + * offline against real threads. This is the one that answers whether quality + * moved, and it cannot be deterministic, so it must not gate a merge. + * + * `SHADOW_MODE` gates only the platform post-back (`ai-response.ts:824`) — + * retrieval and generation run fully either way — so the live mode needs no new + * safety machinery, just a caller that feeds real threads through + * `AIPipeline.generateSupportResponse` and hands the text here. + */ + +import { checkReply, RULES } from './rules.js'; +import type { RuleId, RuleResult } from './rules.js'; +import type { SearchResult } from '../types.js'; + +export interface EvalCase { + /** Stable id, used to diff one run against another. */ + id: string; + /** What the reporter asked, for the report's readability. */ + question: string; + /** The reply under judgement. */ + reply: string; + /** The sources the reply was generated from — the grounding lookup needs these. */ + sources: SearchResult[]; + /** Where this case came from, so a reader can go check it. */ + provenance: string; +} + +export interface CaseScore { + id: string; + results: RuleResult[]; + /** Rules this case broke. Empty means it passed everything. */ + failed: RuleId[]; +} + +export interface EvalReport { + cases: CaseScore[]; + /** Per rule: how many cases passed out of how many were scored. */ + perRule: Record; + /** Cases that broke nothing. */ + cleanCases: number; + totalCases: number; +} + +export function scoreCases(cases: EvalCase[]): EvalReport { + // Refuses an empty set rather than reporting one as clean. With no cases, + // every rule scored `{passed: 0, total: 0}` and `formatReport` printed six + // `ok` lines because `passed === total` — so a live run whose fixture loading + // silently produced nothing rendered as a perfect score. Silence + // indistinguishable from success, in the tool built to detect exactly that. + if (cases.length === 0) { + throw new Error( + 'scoreCases received no cases. An empty set cannot be scored — it would ' + + 'report every rule as passing. Check that the fixtures actually loaded.', + ); + } + + const scored: CaseScore[] = cases.map((c) => { + const results = checkReply(c.reply, c.sources); + return { + id: c.id, + results, + failed: results.filter((r) => !r.passed).map((r) => r.rule), + }; + }); + + const perRule = Object.fromEntries( + RULES.map((rule) => [ + rule, + { + passed: scored.filter((s) => !s.failed.includes(rule)).length, + total: scored.length, + }, + ]), + ) as Record; + + return { + cases: scored, + perRule, + cleanCases: scored.filter((s) => s.failed.length === 0).length, + totalCases: scored.length, + }; +} + +/** Human-readable report, for the offline runs where somebody reads the output. */ +export function formatReport(report: EvalReport): string { + const lines = [`${report.cleanCases}/${report.totalCases} cases clean`, '']; + for (const rule of RULES) { + const { passed, total } = report.perRule[rule]; + lines.push(` ${passed === total ? 'ok ' : 'FAIL'} ${rule}: ${passed}/${total}`); + } + const dirty = report.cases.filter((c) => c.failed.length > 0); + if (dirty.length) { + lines.push('', 'Failing cases:'); + for (const c of dirty) { + lines.push(` ${c.id}`); + for (const r of c.results.filter((r) => !r.passed)) { + lines.push(` - ${r.rule}: ${r.detail}`); + } + } + } + return lines.join('\n'); +} + +const CHAT_DOCS: SearchResult[] = [ + { + title: 'CopilotChat', + content: + 'CopilotChat renders a chat window. Use the `CopilotChat` component with the ' + + '`instructions` prop. Slots let you replace the input via the `input` prop.', + score: 0.9, + sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat', + }, +]; + +/** + * The four failures from the doc's appendix, as replies the rules must catch. + * + * **These are reconstructions, not transcripts.** The doc describes each reply's + * shape and quotes fragments of it; the full original text lives in the linked + * Discord threads and GitHub issue. Each reply below is assembled from what the + * doc states about it, and the `provenance` field links the original so anyone + * can check the reconstruction against the real thing. + * + * What that means for what these prove: they pin that the RULE SET catches each + * documented failure mode. They do not measure the current agent, because they + * are not its current output. Measuring the agent needs the live mode described + * at the top of this file, fed with the real threads. + */ +export const HISTORICAL_FAILURES: EvalCase[] = [ + { + id: 'case-a-deep-agents-subagents', + question: 'Does Deep Agents support subagents?', + // The doc: said it had "no specific timeline information", sent them to + // GitHub to ask, and offered a workaround as a "hypothesis". Subagents + // work today and one code search returns the proof. + reply: + "Great question! I don't have specific timeline information on subagent support " + + 'for Deep Agents. One possibility is that you could work around it by composing ' + + 'agents manually, or the equivalent pattern in your own runtime. ' + + 'I would suggest opening a GitHub discussion so the team can weigh in.', + sources: [], + provenance: + 'https://discord.com/channels/1122926057641742418/1535447155735789708(2026-08-08)', + }, + { + id: 'case-b-version-mixing', + question: 'How do I render the delegation in the chat?', + // The doc: mixed v1 and v2 hooks in one answer and hedged an API name. + reply: + 'You can hook the render path with `useCopilotFabricatedRender` or the equivalent ' + + 'render hook, and install `@copilotkitnext/react` to get the newer surface.', + sources: CHAT_DOCS, + provenance: + 'https://discord.com/channels/1122926057641742418/1313616713647919218/threads/1529599811744043018 (2026-07-22)', + }, + { + id: 'case-c-false-capability-claim', + question: '(maintainer follow-up in thread) Did that fix work for you?', + // The doc: replied to a maintainer, complimented his community spirit, + // claimed it could not see other people's replies, asked for a version. + reply: + 'Thanks for your detailed report and for supporting the community here! ' + + "I can't see other people's replies in this thread, so I don't have the full " + + 'context. Which version of CopilotKit are you using?', + sources: CHAT_DOCS, + provenance: + 'https://discord.com/channels/1122926057641742418/1313616713647919218/threads/1531971013791711342 (2026-08-11)', + }, + { + id: 'case-d-five-paragraphs-of-nothing', + question: '(dependency audit listing two concrete problems)', + // The doc: praise opener, restated both of the reporter's points, a + // "What I can't do from here" section, advice on writing better issues. + reply: + 'Great question, and thanks for this detailed report! To summarise what you have ' + + 'found: first, the manifest and the lockfile disagree about the version. Second, ' + + 'the peer dependency range looks too wide. ' + + "Here is what I can't do from here: I cannot read the source or run the install " + + 'to confirm either point. In the future, please include the full lockfile diff so ' + + 'this is easier to triage. The team will take it from here. ' + + // Padded deliberately so the reply clears the handoff cap, which is + // half of what case D is a fixture FOR. Previously `.repeat(2)` bound + // to the last literal only, so the reply ended with a stray duplicate + // sentence rather than the length the comment claimed. + 'Let me know if any of that needs clarifying and someone will pick it up. '.repeat(3), + sources: CHAT_DOCS, + provenance: 'https://github.com/CopilotKit/CopilotKit/issues/6423', + }, +]; + +/** + * The reply the doc holds up as the target shape, written by a maintainer in + * case A's own thread: verdict, proof, minimum code, one caveat. + * + * Present so the rule set is pinned in both directions. A rule set that only + * ever fires is as useless as one that never does, and this is the case that + * catches an over-eager rule before it starts collapsing good answers into + * handoffs. + */ +export const TARGET_SHAPE: EvalCase = { + id: 'case-a-maintainer-answer', + question: 'Does Deep Agents support subagents?', + reply: + 'Subagents work with Deep Agents today; the docs just do not cover them. Pass them ' + + 'straight to `create_deep_agent`. Deep Agents spawns subagents through its built-in ' + + '`task` tool, which runs as a nested subgraph, and our LangGraph adapter streams those ' + + 'by default, so the delegation shows up in the chat. Render it by hooking the `task` ' + + 'tool. One caveat: be on a recent Python adapter. ' + + 'https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/langgraph/agent.ts', + sources: [ + { + title: 'langgraph/agent.ts', + content: + 'create_deep_agent spawns subagents through the built-in task tool, which runs ' + + 'as a nested subgraph. The LangGraph adapter streams subgraph events by default.', + score: 0.95, + sourceUrl: + 'https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/langgraph/agent.ts', + }, + ], + provenance: + "Maintainer reply quoted in the Agent's Output Doc, from the case-a thread (2026-08-08)", +}; diff --git a/packages/outpost/ai/src/eval/rules.test.ts b/packages/outpost/ai/src/eval/rules.test.ts new file mode 100644 index 0000000..a4f157c --- /dev/null +++ b/packages/outpost/ai/src/eval/rules.test.ts @@ -0,0 +1,208 @@ +import { describe, it, expect } from 'vitest'; +import { checkReply, RULES, HANDOFF_WORD_CAP, MIN_REPLY_WORDS } from './rules.js'; +import type { SearchResult } from '../types.js'; + +const source = (content: string, title = 'CopilotChat'): SearchResult => ({ + title, + content, + score: 0.9, + sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat', +}); + +const DOCS = [source('Use the `CopilotChat` component with the `instructions` prop.')]; + +/** Convenience: the ids of every rule the reply violated. */ +const broken = (reply: string, sources: SearchResult[] = DOCS): string[] => + checkReply(reply, sources) + .filter((r) => !r.passed) + .map((r) => r.rule); + +describe('checkReply', () => { + it('passes a reply that answers with a grounded name and a source', () => { + expect( + broken( + 'Use the `CopilotChat` component with the `instructions` prop. ' + + 'See https://docs.copilotkit.ai/reference/components/chat/CopilotChat', + ), + ).toEqual([]); + }); + + it('reports every rule exactly once, pass or fail', () => { + const results = checkReply('Anything at all.', DOCS); + expect(results.map((r) => r.rule).sort()).toEqual([...RULES].sort()); + expect(new Set(results.map((r) => r.rule)).size).toBe(RULES.length); + }); +}); + +// "Zero invented API names — this one is mechanically checkable, so any +// occurrence is a bug, not a judgment call." Delegates to the same +// assessGroundedness the pipeline gates on, so the harness and production +// cannot disagree about what counts as invented. +describe('grounded-identifiers', () => { + it('fails on a name that appears in none of the sources', () => { + expect(broken('Call `useCopilotFabricated()` to fix it.')).toContain( + 'grounded-identifiers', + ); + }); + + it('passes a name the sources actually contain', () => { + expect(broken('Use the `CopilotChat` component.')).not.toContain('grounded-identifiers'); + }); + + it('names the offending identifiers in the detail, so a failure is actionable', () => { + const result = checkReply('Call `useCopilotFabricated()`.', DOCS).find( + (r) => r.rule === 'grounded-identifiers', + ); + expect(result?.detail).toContain('useCopilotFabricated'); + }); +}); + +// "Every substantive answer links to the doc page or file it came from. If it +// can't, it becomes a two-sentence handoff instead." So the rule is a +// disjunction, not a flat requirement — a short handoff is allowed to have no +// link, and that is the whole point of it existing. +describe('cites-or-is-a-short-handoff', () => { + it('fails a long answer that cites nothing', () => { + const wordy = 'You can configure this in several ways. '.repeat(12); + expect(broken(wordy)).toContain('cites-or-is-a-short-handoff'); + }); + + it('passes a long answer that links the docs', () => { + const wordy = + 'You can configure this in several ways. '.repeat(12) + + ' https://docs.copilotkit.ai/guides/configuration'; + expect(broken(wordy)).not.toContain('cites-or-is-a-short-handoff'); + }); + + it('passes a long answer that links a repo file, since code is a real answer', () => { + const wordy = + 'This is handled by the adapter. '.repeat(12) + + ' https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/agent.ts'; + expect(broken(wordy)).not.toContain('cites-or-is-a-short-handoff'); + }); + + it('passes a short handoff with no link at all', () => { + expect( + broken('Confirmed the manifest and lockfile skew. Routing this to the team.'), + ).not.toContain('cites-or-is-a-short-handoff'); + }); +}); + +// "Nothing found -> two sentences, done." A no-answer that runs to 400 words is +// the single thing the doc says changes the feel of the product most. +describe('the handoff cap', () => { + it('fails a handoff that pads past the cap', () => { + const padded = + 'A human will follow up here shortly. ' + + 'In the meantime here is some general background. '.repeat(20); + const results = checkReply(padded, DOCS); + const rule = results.find((r) => r.rule === 'cites-or-is-a-short-handoff'); + expect(rule?.passed).toBe(false); + expect(rule?.detail).toContain(String(HANDOFF_WORD_CAP)); + }); + + it('does not apply the cap to an answer that carries a source', () => { + const long = + 'Use the `CopilotChat` component. '.repeat(30) + + ' https://docs.copilotkit.ai/reference/components/chat/CopilotChat'; + expect(broken(long)).not.toContain('cites-or-is-a-short-handoff'); + }); +}); + +// Every other rule is a prohibition, so without this one the score is maximised +// by saying nothing — and an agent regressing toward empty replies would read as +// the score improving. +describe('says-something', () => { + it.each(['', ' ', 'No.', 'Escalating.', 'Routing this to the team.'])( + 'fails %o, which is not a reply', + (reply) => { + expect(broken(reply, [])).toContain('says-something'); + }, + ); + + it('passes the shortest reply the doc actually endorses', () => { + // The reference handoff, quoted in the doc as the right answer for case D. + expect( + broken( + 'Confirmed the manifest and lockfile skew. Routing this to the team — someone will follow up here.', + ), + ).toEqual([]); + }); + + it('reports the word count so a failure is actionable', () => { + const result = checkReply('No.', DOCS).find((r) => r.rule === 'says-something'); + expect(result?.detail).toContain('1 words'); + expect(MIN_REPLY_WORDS).toBeGreaterThan(1); + }); +}); + +// "Never mention @copilotkitnext" is right as a default and wrong as an +// absolute: the reporter importing from it needs to be told what to import +// instead, and under a flat ban that reply is the one that cannot be given. +describe('the migration answer', () => { + it('allows naming the dead package when the live one is named too', () => { + expect( + broken( + "You're importing from `@copilotkitnext/react`, which merged into " + + '`@copilotkit/react-core` v2 — switch the import and the hook names carry over.', + ), + ).not.toContain('no-dead-package'); + }); + + it('still fails a stray reference with no replacement named', () => { + expect(broken('Install `@copilotkitnext/react` first and then retry the build.')).toContain( + 'no-dead-package', + ); + }); +}); + +// "Never talk about the agent's own limits", "no praise openers", "never claim it +// can't see the thread", "don't coach people on how to write better issues". +// Case D was five paragraphs of exactly these. +describe('no-banned-phrases', () => { + it.each([ + 'Great question! Use the `CopilotChat` component.', + 'Thanks for this detailed report. Use the `CopilotChat` component.', + "Here is what I can't do from here: read the source.", + "I can't see other people's replies in this thread.", + 'In the future, please include a minimal reproduction in your issue.', + ])('fails on %s', (reply) => { + expect(broken(reply)).toContain('no-banned-phrases'); + }); + + it('passes a reply that just answers', () => { + expect(broken('Use the `CopilotChat` component.')).not.toContain('no-banned-phrases'); + }); + + // The phrase list must not fire on ordinary prose that happens to contain a + // banned word, or the linter collapses correct answers into handoffs. + it('does not fire on innocent uses of the same words', () => { + expect( + broken('The `instructions` prop question comes up often; see the docs.'), + ).not.toContain('no-banned-phrases'); + }); +}); + +// "Never hedge a name. 'or the equivalent hook' means the agent is guessing." +describe('no-hedged-names', () => { + it('fails on a hedged identifier', () => { + expect(broken('Use `CopilotChat` or the equivalent render hook.')).toContain( + 'no-hedged-names', + ); + }); + + it('passes an unhedged one', () => { + expect(broken('Use the `CopilotChat` component.')).not.toContain('no-hedged-names'); + }); +}); + +// "Never mention @copilotkitnext. It's dead — it merged into v2." +describe('no-dead-package', () => { + it('fails on a mention of the retired package', () => { + expect(broken('Install `@copilotkitnext/react` first.')).toContain('no-dead-package'); + }); + + it('passes the live package', () => { + expect(broken('Install `@copilotkit/react-core` first.')).not.toContain('no-dead-package'); + }); +}); diff --git a/packages/outpost/ai/src/eval/rules.ts b/packages/outpost/ai/src/eval/rules.ts new file mode 100644 index 0000000..86f1c50 --- /dev/null +++ b/packages/outpost/ai/src/eval/rules.ts @@ -0,0 +1,234 @@ +/** + * The reply rules from "Fix the Agent's Output", as code. + * + * The doc's own success criteria are deliberately mechanical — *"zero invented + * API names — this one is mechanically checkable, so any occurrence is a bug, + * not a judgment call"* — and this module is that check. Nothing here calls a + * model; every rule is a regex or a set lookup over the reply plus the sources + * it was generated from. + * + * ## Two consumers, one rule set + * + * These rules are needed twice, and the whole point of putting them here is that + * the two uses cannot drift apart: + * + * 1. **The eval harness** scores a reply after the fact, to answer "did answer + * quality move?" across a fixture set of real threads. + * 2. **The draft linter** (the doc's step 3) runs the same rules *before* the + * reply posts, and a failure collapses the draft into the two-sentence + * handoff rather than being cleaned up and published. + * + * If the linter and the harness ever disagreed about what "invented API name" + * means, the score would stop predicting the behaviour. So the harness measures + * exactly what the linter will enforce. + * + * ## What is deliberately NOT here + * + * Reply-type classification (Answer / Partial / Route / Silent). That taxonomy + * does not exist in the code yet, and inferring it from the finished text is + * guesswork — the reply type is chosen from the *evidence*, before writing, so + * only the pipeline can report it honestly. Until it does, the rules below score + * observable properties (does it cite, how long is it) rather than pretending to + * recover the decision. + */ + +import { assessGroundedness } from '../groundedness.js'; +import type { SearchResult } from '../types.js'; + +/** + * Word cap on a reply that cites nothing. + * + * The doc's number. A no-answer is currently ~400 words of hedging and should be + * one line, so the cap is what makes "nothing found -> two sentences, done" + * checkable rather than aspirational. + */ +export const HANDOFF_WORD_CAP = 60; + +/** + * Fewest words that can count as a reply at all. + * + * Every other rule here is a prohibition, so without this one the score is + * MAXIMISED by saying nothing: `checkReply('')` passed all of them, and so did + * `'No.'` and `'Escalating.'`. In the live mode that matters most — an agent + * regressing toward empty or near-empty replies would show up as the score + * IMPROVING, in the tool built to catch exactly that. + * + * Set against the doc's own floor rather than picked: the shortest acceptable + * reply is a Route, and a Route is *"two sentences. What we confirmed, if + * anything, and that a human is picking it up."* The reference handoff in the + * doc — "Confirmed the manifest/lockfile skew. Routing this to the team — + * someone will follow up here." — is 15 words, so 8 leaves real headroom while + * still rejecting a bare acknowledgement. + * + * A SILENT reply is not a short reply, it is no reply, and is never scored here. + */ +export const MIN_REPLY_WORDS = 8; + +/** + * Phrases the reply may never contain, each traceable to a case in the doc. + * + * Anchored tightly on purpose. A rule that fires on ordinary prose is worse than + * no rule, because a linter failure collapses the draft into a handoff — so a + * false positive here costs a reporter a correct answer, which is the same + * failure direction as the groundedness gate withholding one. + */ +const BANNED_PHRASES: Array<{ pattern: RegExp; why: string }> = [ + // Case D's opener, and the doc's "no praise openers" rule. + { pattern: /\bgreat question\b/i, why: 'praise opener' }, + { + pattern: /\bthanks for (?:this|the|your) (?:detailed |thorough |thoughtful )?report\b/i, + why: 'praise opener', + }, + { pattern: /\bexcellent (?:question|report|catch)\b/i, why: 'praise opener' }, + // Case D's "What I can't do from here" section, and the rule against the + // agent performing its own humility. + { pattern: /\bwhat i (?:can'?t|cannot) do\b/i, why: 'self-commentary about its own limits' }, + { + pattern: /\bi (?:haven'?t|have not) read the source\b/i, + why: 'self-commentary about its own limits', + }, + { + pattern: /\bi (?:don'?t|do not) have access to\b/i, + why: 'self-commentary about its own limits', + }, + // Case C: it claimed it could not read the thread. It can. + { + pattern: + /\bi (?:can'?t|cannot) see (?:other|the other|anyone)[^.]{0,40}\b(?:replies|messages|responses)\b/i, + why: 'false claim that it cannot see the thread', + }, + // Case D again: coaching the reporter on how to file better issues. + { + pattern: /\bin the future,? please (?:include|provide|attach|add)\b/i, + why: 'coaching the reporter on how to write issues', + }, +]; + +/** "Never hedge a name" — a hedge means the name is a guess, so it must go. */ +const HEDGED_NAME_PATTERNS: RegExp[] = [ + /\bor the equivalent\b/i, + /\bor (?:its|the) equivalent\b/i, + /\bor something similar\b/i, +]; + +/** + * The retired package. + * + * `@copilotkitnext` was the useAgent-era v2 line and merged into `@copilotkit` + * v2. Naming it sends a reporter to a package that no longer exists, so the doc + * makes this an absolute: never mentioned. + */ +const DEAD_PACKAGE = /@copilotkitnext\b/i; + +/** + * The one reply that legitimately names the retired package. + * + * "Never mention `@copilotkitnext`" is right as a default and wrong as an + * absolute: someone importing from it needs to be told what to import instead, + * and *"you're importing from `@copilotkitnext/react`, which merged into + * `@copilotkit/react-core` v2 — switch the import"* is the correct answer. Under + * a flat ban that reply fails, and in the linter it collapses into a handoff — + * so the one reporter who most needs the migration answer is the only one who + * cannot get it. + * + * The carve-out is narrow: naming the dead package is allowed only when the + * reply also names a live `@copilotkit/` package, which is what makes it a + * migration instruction rather than a stray reference. + */ +const LIVE_PACKAGE = /@copilotkit\/[a-z-]+/i; + +/** A link that constitutes a citation: a docs page or a file in the repo. */ +const CITATION_LINK = + /https?:\/\/(?:[a-z0-9-]+\.)*(?:copilotkit\.ai|github\.com\/CopilotKit|github\.com\/ag-ui-protocol)\/\S+/i; + +export const RULES = [ + 'says-something', + 'grounded-identifiers', + 'cites-or-is-a-short-handoff', + 'no-banned-phrases', + 'no-hedged-names', + 'no-dead-package', +] as const; + +export type RuleId = (typeof RULES)[number]; + +export interface RuleResult { + rule: RuleId; + passed: boolean; + /** Why it failed, naming the offending text. Empty when it passed. */ + detail: string; +} + +function countWords(text: string): number { + const trimmed = text.trim(); + return trimmed ? trimmed.split(/\s+/).length : 0; +} + +/** + * Run every rule against one reply. Always returns one result per rule, so a + * report can distinguish "passed" from "not evaluated". + * + * `sources` must be the results the reply was actually generated from — the + * grounding rule is a lookup against them, so passing a different set silently + * turns the strictest rule into a no-op. + */ +export function checkReply(reply: string, sources: SearchResult[]): RuleResult[] { + const words = countWords(reply); + const cites = CITATION_LINK.test(reply); + + // A reply that cites nothing is only acceptable as a short handoff, so the + // two rules below are the two halves of that single sentence in the doc. + const isShortEnoughForHandoff = words <= HANDOFF_WORD_CAP; + + const groundedness = assessGroundedness(reply, sources); + const banned = BANNED_PHRASES.filter(({ pattern }) => pattern.test(reply)); + const hedged = HEDGED_NAME_PATTERNS.filter((pattern) => pattern.test(reply)); + + return [ + { + rule: 'says-something', + passed: words >= MIN_REPLY_WORDS, + detail: + words >= MIN_REPLY_WORDS + ? '' + : `${words} words — too short to be a reply; the shortest acceptable one is a two-sentence handoff`, + }, + { + rule: 'grounded-identifiers', + passed: groundedness.unsourcedIdentifiers.length === 0, + detail: groundedness.unsourcedIdentifiers.length + ? `names not present in any source: ${groundedness.unsourcedIdentifiers.join(', ')}` + : '', + }, + { + // One rule, not two. `source-link-or-handoff` and `handoff-is-short` + // evaluated the identical expression, so they could never disagree — + // which presented five independent signals as six and double-counted + // every failure in both the per-rule table and the report. + rule: 'cites-or-is-a-short-handoff', + passed: cites || isShortEnoughForHandoff, + detail: + cites || isShortEnoughForHandoff + ? '' + : `${words} words with no docs or repo link, over the ${HANDOFF_WORD_CAP}-word handoff cap; a reply this long has to cite what it came from`, + }, + { + rule: 'no-banned-phrases', + passed: banned.length === 0, + detail: banned.map(({ why }) => why).join('; '), + }, + { + rule: 'no-hedged-names', + passed: hedged.length === 0, + detail: hedged.length ? 'hedges an API name, which means it is guessing' : '', + }, + { + rule: 'no-dead-package', + passed: !DEAD_PACKAGE.test(reply) || LIVE_PACKAGE.test(reply), + detail: + DEAD_PACKAGE.test(reply) && !LIVE_PACKAGE.test(reply) + ? 'mentions @copilotkitnext without naming the @copilotkit/ package that replaced it' + : '', + }, + ]; +} diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 75857e1..071cbcd 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -47,4 +47,14 @@ export type { TopIssueInput, ScoredTopIssue, } from './front-door.js'; +// The rule set and the scorer are API — Phase 3's linter consumes them. +// HISTORICAL_FAILURES and TARGET_SHAPE deliberately are NOT: they are test data, +// and exporting them put reconstructed bad replies containing +// `useCopilotFabricatedRender` and `@copilotkitnext/react` into dist and the +// worker image, where they would surface in a bundle grep for the dead package. +// Import them from './eval/harness.js' directly in tests and offline runners. +export { checkReply, RULES, HANDOFF_WORD_CAP, MIN_REPLY_WORDS } from './eval/rules.js'; +export type { RuleId, RuleResult } from './eval/rules.js'; +export { scoreCases, formatReport } from './eval/harness.js'; +export type { EvalCase, CaseScore, EvalReport } from './eval/harness.js'; export * from './types.js';