From ccbf95334cf6f01eb5ecc251c827492968c41009 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?=
 <66887028+NathanTarbert@users.noreply.github.com>
Date: Sun, 23 Aug 2026 07:01:55 -0400
Subject: [PATCH 1/3] feat(ai): score a reply against the response-quality
 rules
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Phase 1 of the response-quality work (CPK-8076). The Agent's Output Doc
lists its success criteria as deliberately mechanical — "zero invented API
names, this one is mechanically checkable, so any occurrence is a bug, not
a judgment call" — and this is that check, plus the aggregation that turns
it into a number.

Six rules, each traceable to a line in the doc: identifiers must appear in
the retrieved sources, a reply either cites or stays under the 60-word
handoff cap, no praise openers or self-commentary or false "I can't see the
thread" claims, no hedged API names, no mention of the retired
@copilotkitnext.

One rule set, two consumers, and that is the reason it lives in its own
module rather than inside the harness: the doc's step 3 linter runs the
same rules BEFORE a reply posts, collapsing a failing draft into the
two-sentence handoff. If the linter and the harness disagreed about what
"invented API name" means, the score would stop predicting the behaviour.
The grounding rule delegates to assessGroundedness for the same reason —
production already gates on it.

The golden set is the doc's four appendix failures, and all four are caught.
They are RECONSTRUCTIONS from the doc's descriptions and quoted fragments,
not transcripts; each carries a provenance link to the original thread so
the reconstruction can be checked. So they pin that the rule set catches
each documented failure mode. They do not measure the current agent, which
needs the live mode: real threads through AIPipeline.generateSupportResponse,
scored by the same rules. SHADOW_MODE gates only the platform post-back, so
that needs no new safety machinery, just a caller.

The maintainer reply the doc holds up as correct is included and passes all
six. A rule set that fires on everything is as useless as one that never
fires, and a false positive here costs a reporter a correct answer — the
same failure direction as the groundedness gate withholding one.

Known thinness, measured rather than assumed: grounded-identifiers only
fires on case B. Cases A, C and D name no CopilotKit-shaped identifier, so
the strictest rule passes vacuously on three of the four.

57 tests. Mutation-checked rather than trusted green: forcing
grounded-identifiers to pass, loosening the praise-opener pattern to bare
/question/i, and dropping the cites-or-short disjunction each kill the tests
that name them.

Refs CPK-8076
---
 packages/outpost/ai/src/eval/harness.test.ts |  85 +++++++
 packages/outpost/ai/src/eval/harness.ts      | 223 +++++++++++++++++++
 packages/outpost/ai/src/eval/rules.test.ts   | 161 +++++++++++++
 packages/outpost/ai/src/eval/rules.ts        | 184 +++++++++++++++
 packages/outpost/ai/src/index.ts             |   4 +
 5 files changed, 657 insertions(+)
 create mode 100644 packages/outpost/ai/src/eval/harness.test.ts
 create mode 100644 packages/outpost/ai/src/eval/harness.ts
 create mode 100644 packages/outpost/ai/src/eval/rules.test.ts
 create mode 100644 packages/outpost/ai/src/eval/rules.ts

diff --git a/packages/outpost/ai/src/eval/harness.test.ts b/packages/outpost/ai/src/eval/harness.test.ts
new file mode 100644
index 00000000..b9dd60f8
--- /dev/null
+++ b/packages/outpost/ai/src/eval/harness.test.ts
@@ -0,0 +1,85 @@
+import { describe, it, expect } from 'vitest';
+import {
+    scoreCases,
+    formatReport,
+    HISTORICAL_FAILURES,
+    TARGET_SHAPE,
+} from './harness.js';
+import { RULES } from './rules.js';
+
+describe('scoreCases', () => {
+    it('reports every rule for every case', () => {
+        const report = scoreCases(HISTORICAL_FAILURES);
+        expect(report.totalCases).toBe(HISTORICAL_FAILURES.length);
+        for (const rule of RULES) {
+            expect(report.perRule[rule].total).toBe(HISTORICAL_FAILURES.length);
+        }
+    });
+
+    it('counts a clean case as clean', () => {
+        const report = scoreCases([TARGET_SHAPE]);
+        expect(report.cleanCases).toBe(1);
+        expect(report.cases[0].failed).toEqual([]);
+    });
+});
+
+// The point of the golden set: every documented failure has to be caught by at
+// least one rule. If a rule regresses, one of these stops failing — and a case
+// that stops failing is exactly as alarming as a test that stops passing.
+describe('the documented failures are all caught', () => {
+    it.each(HISTORICAL_FAILURES.map((c) => [c.id, c] as const))(
+        '%s breaks at least one rule',
+        (_id, testCase) => {
+            const report = scoreCases([testCase]);
+            expect(report.cases[0].failed.length).toBeGreaterThan(0);
+        },
+    );
+
+    // Named individually rather than only in aggregate, so a regression says
+    // WHICH failure mode stopped being caught.
+    it('catches case A for hedging a name and for citing nothing', () => {
+        const failed = scoreCases([HISTORICAL_FAILURES[0]]).cases[0].failed;
+        expect(failed).toContain('no-hedged-names');
+        expect(failed).toContain('no-banned-phrases');
+    });
+
+    it('catches case B for the dead package and the invented name', () => {
+        const failed = scoreCases([HISTORICAL_FAILURES[1]]).cases[0].failed;
+        expect(failed).toContain('no-dead-package');
+        expect(failed).toContain('grounded-identifiers');
+    });
+
+    it('catches case C for the false capability claim', () => {
+        const failed = scoreCases([HISTORICAL_FAILURES[2]]).cases[0].failed;
+        expect(failed).toContain('no-banned-phrases');
+    });
+
+    it('catches case D for self-commentary and issue-writing advice', () => {
+        const failed = scoreCases([HISTORICAL_FAILURES[3]]).cases[0].failed;
+        expect(failed).toContain('no-banned-phrases');
+    });
+});
+
+// A rule set that fires on everything is as useless as one that never fires, and
+// this is the direction that costs a reporter a correct answer.
+describe('the target shape passes', () => {
+    it('does not flag the maintainer answer the doc holds up as correct', () => {
+        const report = scoreCases([TARGET_SHAPE]);
+        expect(report.cases[0].failed).toEqual([]);
+    });
+});
+
+describe('formatReport', () => {
+    it('leads with the clean count and names each failing rule', () => {
+        const text = formatReport(scoreCases(HISTORICAL_FAILURES));
+        expect(text).toContain(`0/${HISTORICAL_FAILURES.length} cases clean`);
+        expect(text).toContain('no-dead-package');
+        expect(text).toContain('Failing cases:');
+    });
+
+    it('says nothing about failing cases when there are none', () => {
+        const text = formatReport(scoreCases([TARGET_SHAPE]));
+        expect(text).toContain('1/1 cases clean');
+        expect(text).not.toContain('Failing cases:');
+    });
+});
diff --git a/packages/outpost/ai/src/eval/harness.ts b/packages/outpost/ai/src/eval/harness.ts
new file mode 100644
index 00000000..12a117d0
--- /dev/null
+++ b/packages/outpost/ai/src/eval/harness.ts
@@ -0,0 +1,223 @@
+/**
+ * Scoring and aggregation for the response-quality eval.
+ *
+ * `rules.ts` judges one reply. This turns a set of judged replies into the
+ * numbers the doc asks for — a per-rule pass rate, and the list of cases that
+ * regressed — so "did answer quality move?" has an answer that isn't someone's
+ * impression of the last few threads they read.
+ *
+ * ## How this gets driven
+ *
+ * `scoreCases` takes replies that were *already produced*. It deliberately does
+ * not call the pipeline itself, because the two ways of producing a reply have
+ * opposite requirements and only one belongs in CI:
+ *
+ *   - **Fixed replies** (this file's `HISTORICAL_FAILURES`) — deterministic, no
+ *     network, no model. Pins the rule set against known-bad output so a rule
+ *     cannot silently stop firing. Runs in CI.
+ *   - **Live replies** — real Pathfinder retrieval and a real model call, run
+ *     offline against real threads. This is the one that answers whether quality
+ *     moved, and it cannot be deterministic, so it must not gate a merge.
+ *
+ * `SHADOW_MODE` gates only the platform post-back (`ai-response.ts:178`) —
+ * retrieval and generation run fully either way — so the live mode needs no new
+ * safety machinery, just a caller that feeds real threads through
+ * `AIPipeline.generateSupportResponse` and hands the text here.
+ */
+
+import { checkReply, RULES } from './rules.js';
+import type { RuleId, RuleResult } from './rules.js';
+import type { SearchResult } from '../types.js';
+
+export interface EvalCase {
+    /** Stable id, used to diff one run against another. */
+    id: string;
+    /** What the reporter asked, for the report's readability. */
+    question: string;
+    /** The reply under judgement. */
+    reply: string;
+    /** The sources the reply was generated from — the grounding lookup needs these. */
+    sources: SearchResult[];
+    /** Where this case came from, so a reader can go check it. */
+    provenance: string;
+}
+
+export interface CaseScore {
+    id: string;
+    results: RuleResult[];
+    /** Rules this case broke. Empty means it passed everything. */
+    failed: RuleId[];
+}
+
+export interface EvalReport {
+    cases: CaseScore[];
+    /** Per rule: how many cases passed out of how many were scored. */
+    perRule: Record<RuleId, { passed: number; total: number }>;
+    /** Cases that broke nothing. */
+    cleanCases: number;
+    totalCases: number;
+}
+
+export function scoreCases(cases: EvalCase[]): EvalReport {
+    const scored: CaseScore[] = cases.map((c) => {
+        const results = checkReply(c.reply, c.sources);
+        return {
+            id: c.id,
+            results,
+            failed: results.filter((r) => !r.passed).map((r) => r.rule),
+        };
+    });
+
+    const perRule = Object.fromEntries(
+        RULES.map((rule) => [
+            rule,
+            {
+                passed: scored.filter((s) => !s.failed.includes(rule)).length,
+                total: scored.length,
+            },
+        ]),
+    ) as Record<RuleId, { passed: number; total: number }>;
+
+    return {
+        cases: scored,
+        perRule,
+        cleanCases: scored.filter((s) => s.failed.length === 0).length,
+        totalCases: scored.length,
+    };
+}
+
+/** Human-readable report, for the offline runs where somebody reads the output. */
+export function formatReport(report: EvalReport): string {
+    const lines = [`${report.cleanCases}/${report.totalCases} cases clean`, ''];
+    for (const rule of RULES) {
+        const { passed, total } = report.perRule[rule];
+        lines.push(`  ${passed === total ? 'ok  ' : 'FAIL'} ${rule}: ${passed}/${total}`);
+    }
+    const dirty = report.cases.filter((c) => c.failed.length > 0);
+    if (dirty.length) {
+        lines.push('', 'Failing cases:');
+        for (const c of dirty) {
+            lines.push(`  ${c.id}`);
+            for (const r of c.results.filter((r) => !r.passed)) {
+                lines.push(`    - ${r.rule}: ${r.detail}`);
+            }
+        }
+    }
+    return lines.join('\n');
+}
+
+const CHAT_DOCS: SearchResult[] = [
+    {
+        title: 'CopilotChat',
+        content:
+            'CopilotChat renders a chat window. Use the `CopilotChat` component with the ' +
+            '`instructions` prop. Slots let you replace the input via the `input` prop.',
+        score: 0.9,
+        sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat',
+    },
+];
+
+/**
+ * The four failures from the doc's appendix, as replies the rules must catch.
+ *
+ * **These are reconstructions, not transcripts.** The doc describes each reply's
+ * shape and quotes fragments of it; the full original text lives in the linked
+ * Discord threads and GitHub issue. Each reply below is assembled from what the
+ * doc states about it, and the `provenance` field links the original so anyone
+ * can check the reconstruction against the real thing.
+ *
+ * What that means for what these prove: they pin that the RULE SET catches each
+ * documented failure mode. They do not measure the current agent, because they
+ * are not its current output. Measuring the agent needs the live mode described
+ * at the top of this file, fed with the real threads.
+ */
+export const HISTORICAL_FAILURES: EvalCase[] = [
+    {
+        id: 'case-a-deep-agents-subagents',
+        question: 'Does Deep Agents support subagents?',
+        // The doc: said it had "no specific timeline information", sent them to
+        // GitHub to ask, and offered a workaround as a "hypothesis". Subagents
+        // work today and one code search returns the proof.
+        reply:
+            "Great question! I don't have specific timeline information on subagent support " +
+            'for Deep Agents. One possibility is that you could work around it by composing ' +
+            'agents manually, or the equivalent pattern in your own runtime. ' +
+            'I would suggest opening a GitHub discussion so the team can weigh in.',
+        sources: [],
+        provenance:
+            'https://discord.com/channels/1122926057641742418/1535447155735789708(2026-08-08)',
+    },
+    {
+        id: 'case-b-version-mixing',
+        question: 'How do I render the delegation in the chat?',
+        // The doc: mixed v1 and v2 hooks in one answer and hedged an API name.
+        reply:
+            'You can hook the render path with `useCopilotFabricatedRender` or the equivalent ' +
+            'render hook, and install `@copilotkitnext/react` to get the newer surface.',
+        sources: CHAT_DOCS,
+        provenance:
+            'https://discord.com/channels/1122926057641742418/1313616713647919218/threads/1529599811744043018 (2026-07-22)',
+    },
+    {
+        id: 'case-c-false-capability-claim',
+        question: '(maintainer follow-up in thread) Did that fix work for you?',
+        // The doc: replied to a maintainer, complimented his community spirit,
+        // claimed it could not see other people's replies, asked for a version.
+        reply:
+            'Thanks for your detailed report and for supporting the community here! ' +
+            "I can't see other people's replies in this thread, so I don't have the full " +
+            'context. Which version of CopilotKit are you using?',
+        sources: CHAT_DOCS,
+        provenance:
+            'https://discord.com/channels/1122926057641742418/1313616713647919218/threads/1531971013791711342 (2026-08-11)',
+    },
+    {
+        id: 'case-d-five-paragraphs-of-nothing',
+        question: '(dependency audit listing two concrete problems)',
+        // The doc: praise opener, restated both of the reporter's points, a
+        // "What I can't do from here" section, advice on writing better issues.
+        reply:
+            'Great question, and thanks for this detailed report! To summarise what you have ' +
+            'found: first, the manifest and the lockfile disagree about the version. Second, ' +
+            'the peer dependency range looks too wide. ' +
+            "Here is what I can't do from here: I cannot read the source or run the install " +
+            'to confirm either point. In the future, please include the full lockfile diff so ' +
+            'this is easier to triage. The team will take it from here. '.repeat(2),
+        sources: CHAT_DOCS,
+        provenance: 'https://github.com/CopilotKit/CopilotKit/issues/6423',
+    },
+];
+
+/**
+ * The reply the doc holds up as the target shape, written by a maintainer in
+ * case A's own thread: verdict, proof, minimum code, one caveat.
+ *
+ * Present so the rule set is pinned in both directions. A rule set that only
+ * ever fires is as useless as one that never does, and this is the case that
+ * catches an over-eager rule before it starts collapsing good answers into
+ * handoffs.
+ */
+export const TARGET_SHAPE: EvalCase = {
+    id: 'case-a-maintainer-answer',
+    question: 'Does Deep Agents support subagents?',
+    reply:
+        'Subagents work with Deep Agents today; the docs just do not cover them. Pass them ' +
+        'straight to `create_deep_agent`. Deep Agents spawns subagents through its built-in ' +
+        '`task` tool, which runs as a nested subgraph, and our LangGraph adapter streams those ' +
+        'by default, so the delegation shows up in the chat. Render it by hooking the `task` ' +
+        'tool. One caveat: be on a recent Python adapter. ' +
+        'https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/langgraph/agent.ts',
+    sources: [
+        {
+            title: 'langgraph/agent.ts',
+            content:
+                'create_deep_agent spawns subagents through the built-in task tool, which runs ' +
+                'as a nested subgraph. The LangGraph adapter streams subgraph events by default.',
+            score: 0.95,
+            sourceUrl:
+                'https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/langgraph/agent.ts',
+        },
+    ],
+    provenance:
+        'Maintainer reply quoted in the Agent\'s Output Doc, from the case-a thread (2026-08-08)',
+};
diff --git a/packages/outpost/ai/src/eval/rules.test.ts b/packages/outpost/ai/src/eval/rules.test.ts
new file mode 100644
index 00000000..e33718cb
--- /dev/null
+++ b/packages/outpost/ai/src/eval/rules.test.ts
@@ -0,0 +1,161 @@
+import { describe, it, expect } from 'vitest';
+import { checkReply, RULES, HANDOFF_WORD_CAP } from './rules.js';
+import type { SearchResult } from '../types.js';
+
+const source = (content: string, title = 'CopilotChat'): SearchResult => ({
+    title,
+    content,
+    score: 0.9,
+    sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat',
+});
+
+const DOCS = [source('Use the `CopilotChat` component with the `instructions` prop.')];
+
+/** Convenience: the ids of every rule the reply violated. */
+const broken = (reply: string, sources: SearchResult[] = DOCS): string[] =>
+    checkReply(reply, sources)
+        .filter((r) => !r.passed)
+        .map((r) => r.rule);
+
+describe('checkReply', () => {
+    it('passes a reply that answers with a grounded name and a source', () => {
+        expect(
+            broken(
+                'Use the `CopilotChat` component with the `instructions` prop. ' +
+                    'See https://docs.copilotkit.ai/reference/components/chat/CopilotChat',
+            ),
+        ).toEqual([]);
+    });
+
+    it('reports every rule exactly once, pass or fail', () => {
+        const results = checkReply('Anything at all.', DOCS);
+        expect(results.map((r) => r.rule).sort()).toEqual([...RULES].sort());
+        expect(new Set(results.map((r) => r.rule)).size).toBe(RULES.length);
+    });
+});
+
+// "Zero invented API names — this one is mechanically checkable, so any
+// occurrence is a bug, not a judgment call." Delegates to the same
+// assessGroundedness the pipeline gates on, so the harness and production
+// cannot disagree about what counts as invented.
+describe('grounded-identifiers', () => {
+    it('fails on a name that appears in none of the sources', () => {
+        expect(broken('Call `useCopilotFabricated()` to fix it.')).toContain(
+            'grounded-identifiers',
+        );
+    });
+
+    it('passes a name the sources actually contain', () => {
+        expect(broken('Use the `CopilotChat` component.')).not.toContain('grounded-identifiers');
+    });
+
+    it('names the offending identifiers in the detail, so a failure is actionable', () => {
+        const result = checkReply('Call `useCopilotFabricated()`.', DOCS).find(
+            (r) => r.rule === 'grounded-identifiers',
+        );
+        expect(result?.detail).toContain('useCopilotFabricated');
+    });
+});
+
+// "Every substantive answer links to the doc page or file it came from. If it
+// can't, it becomes a two-sentence handoff instead." So the rule is a
+// disjunction, not a flat requirement — a short handoff is allowed to have no
+// link, and that is the whole point of it existing.
+describe('source-link-or-handoff', () => {
+    it('fails a long answer that cites nothing', () => {
+        const wordy = 'You can configure this in several ways. '.repeat(12);
+        expect(broken(wordy)).toContain('source-link-or-handoff');
+    });
+
+    it('passes a long answer that links the docs', () => {
+        const wordy =
+            'You can configure this in several ways. '.repeat(12) +
+            ' https://docs.copilotkit.ai/guides/configuration';
+        expect(broken(wordy)).not.toContain('source-link-or-handoff');
+    });
+
+    it('passes a long answer that links a repo file, since code is a real answer', () => {
+        const wordy =
+            'This is handled by the adapter. '.repeat(12) +
+            ' https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/agent.ts';
+        expect(broken(wordy)).not.toContain('source-link-or-handoff');
+    });
+
+    it('passes a short handoff with no link at all', () => {
+        expect(
+            broken('Confirmed the manifest and lockfile skew. Routing this to the team.'),
+        ).not.toContain('source-link-or-handoff');
+    });
+});
+
+// "Nothing found -> two sentences, done." A no-answer that runs to 400 words is
+// the single thing the doc says changes the feel of the product most.
+describe('handoff-is-short', () => {
+    it('fails a handoff that pads past the cap', () => {
+        const padded =
+            'A human will follow up here shortly. ' +
+            'In the meantime here is some general background. '.repeat(20);
+        const results = checkReply(padded, DOCS);
+        const handoff = results.find((r) => r.rule === 'handoff-is-short');
+        expect(handoff?.passed).toBe(false);
+        expect(handoff?.detail).toContain(String(HANDOFF_WORD_CAP));
+    });
+
+    it('does not apply the cap to an answer that carries a source', () => {
+        const long =
+            'Use the `CopilotChat` component. '.repeat(30) +
+            ' https://docs.copilotkit.ai/reference/components/chat/CopilotChat';
+        expect(broken(long)).not.toContain('handoff-is-short');
+    });
+});
+
+// "Never talk about the agent's own limits", "no praise openers", "never claim it
+// can't see the thread", "don't coach people on how to write better issues".
+// Case D was five paragraphs of exactly these.
+describe('no-banned-phrases', () => {
+    it.each([
+        'Great question! Use the `CopilotChat` component.',
+        "Thanks for this detailed report. Use the `CopilotChat` component.",
+        "Here is what I can't do from here: read the source.",
+        "I can't see other people's replies in this thread.",
+        'In the future, please include a minimal reproduction in your issue.',
+    ])('fails on %s', (reply) => {
+        expect(broken(reply)).toContain('no-banned-phrases');
+    });
+
+    it('passes a reply that just answers', () => {
+        expect(broken('Use the `CopilotChat` component.')).not.toContain('no-banned-phrases');
+    });
+
+    // The phrase list must not fire on ordinary prose that happens to contain a
+    // banned word, or the linter collapses correct answers into handoffs.
+    it('does not fire on innocent uses of the same words', () => {
+        expect(
+            broken('The `instructions` prop question comes up often; see the docs.'),
+        ).not.toContain('no-banned-phrases');
+    });
+});
+
+// "Never hedge a name. 'or the equivalent hook' means the agent is guessing."
+describe('no-hedged-names', () => {
+    it('fails on a hedged identifier', () => {
+        expect(broken('Use `CopilotChat` or the equivalent render hook.')).toContain(
+            'no-hedged-names',
+        );
+    });
+
+    it('passes an unhedged one', () => {
+        expect(broken('Use the `CopilotChat` component.')).not.toContain('no-hedged-names');
+    });
+});
+
+// "Never mention @copilotkitnext. It's dead — it merged into v2."
+describe('no-dead-package', () => {
+    it('fails on a mention of the retired package', () => {
+        expect(broken('Install `@copilotkitnext/react` first.')).toContain('no-dead-package');
+    });
+
+    it('passes the live package', () => {
+        expect(broken('Install `@copilotkit/react-core` first.')).not.toContain('no-dead-package');
+    });
+});
diff --git a/packages/outpost/ai/src/eval/rules.ts b/packages/outpost/ai/src/eval/rules.ts
new file mode 100644
index 00000000..05ca8177
--- /dev/null
+++ b/packages/outpost/ai/src/eval/rules.ts
@@ -0,0 +1,184 @@
+/**
+ * The reply rules from "Fix the Agent's Output", as code.
+ *
+ * The doc's own success criteria are deliberately mechanical — *"zero invented
+ * API names — this one is mechanically checkable, so any occurrence is a bug,
+ * not a judgment call"* — and this module is that check. Nothing here calls a
+ * model; every rule is a regex or a set lookup over the reply plus the sources
+ * it was generated from.
+ *
+ * ## Two consumers, one rule set
+ *
+ * These rules are needed twice, and the whole point of putting them here is that
+ * the two uses cannot drift apart:
+ *
+ * 1. **The eval harness** scores a reply after the fact, to answer "did answer
+ *    quality move?" across a fixture set of real threads.
+ * 2. **The draft linter** (the doc's step 3) runs the same rules *before* the
+ *    reply posts, and a failure collapses the draft into the two-sentence
+ *    handoff rather than being cleaned up and published.
+ *
+ * If the linter and the harness ever disagreed about what "invented API name"
+ * means, the score would stop predicting the behaviour. So the harness measures
+ * exactly what the linter will enforce.
+ *
+ * ## What is deliberately NOT here
+ *
+ * Reply-type classification (Answer / Partial / Route / Silent). That taxonomy
+ * does not exist in the code yet, and inferring it from the finished text is
+ * guesswork — the reply type is chosen from the *evidence*, before writing, so
+ * only the pipeline can report it honestly. Until it does, the rules below score
+ * observable properties (does it cite, how long is it) rather than pretending to
+ * recover the decision.
+ */
+
+import { assessGroundedness } from '../groundedness.js';
+import type { SearchResult } from '../types.js';
+
+/**
+ * Word cap on a reply that cites nothing.
+ *
+ * The doc's number. A no-answer is currently ~400 words of hedging and should be
+ * one line, so the cap is what makes "nothing found -> two sentences, done"
+ * checkable rather than aspirational.
+ */
+export const HANDOFF_WORD_CAP = 60;
+
+/**
+ * Phrases the reply may never contain, each traceable to a case in the doc.
+ *
+ * Anchored tightly on purpose. A rule that fires on ordinary prose is worse than
+ * no rule, because a linter failure collapses the draft into a handoff — so a
+ * false positive here costs a reporter a correct answer, which is the same
+ * failure direction as the groundedness gate withholding one.
+ */
+const BANNED_PHRASES: Array<{ pattern: RegExp; why: string }> = [
+    // Case D's opener, and the doc's "no praise openers" rule.
+    { pattern: /\bgreat question\b/i, why: 'praise opener' },
+    { pattern: /\bthanks for (?:this|the|your) (?:detailed |thorough |thoughtful )?report\b/i, why: 'praise opener' },
+    { pattern: /\bexcellent (?:question|report|catch)\b/i, why: 'praise opener' },
+    // Case D's "What I can't do from here" section, and the rule against the
+    // agent performing its own humility.
+    { pattern: /\bwhat i (?:can'?t|cannot) do\b/i, why: 'self-commentary about its own limits' },
+    { pattern: /\bi (?:haven'?t|have not) read the source\b/i, why: 'self-commentary about its own limits' },
+    { pattern: /\bi (?:don'?t|do not) have access to\b/i, why: 'self-commentary about its own limits' },
+    // Case C: it claimed it could not read the thread. It can.
+    {
+        pattern: /\bi (?:can'?t|cannot) see (?:other|the other|anyone)[^.]{0,40}\b(?:replies|messages|responses)\b/i,
+        why: 'false claim that it cannot see the thread',
+    },
+    // Case D again: coaching the reporter on how to file better issues.
+    {
+        pattern: /\bin the future,? please (?:include|provide|attach|add)\b/i,
+        why: 'coaching the reporter on how to write issues',
+    },
+];
+
+/** "Never hedge a name" — a hedge means the name is a guess, so it must go. */
+const HEDGED_NAME_PATTERNS: RegExp[] = [
+    /\bor the equivalent\b/i,
+    /\bor (?:its|the) equivalent\b/i,
+    /\bor something similar\b/i,
+];
+
+/**
+ * The retired package.
+ *
+ * `@copilotkitnext` was the useAgent-era v2 line and merged into `@copilotkit`
+ * v2. Naming it sends a reporter to a package that no longer exists, so the doc
+ * makes this an absolute: never mentioned.
+ */
+const DEAD_PACKAGE = /@copilotkitnext\b/i;
+
+/** A link that constitutes a citation: a docs page or a file in the repo. */
+const CITATION_LINK =
+    /https?:\/\/(?:[a-z0-9-]+\.)*(?:copilotkit\.ai|github\.com\/CopilotKit|github\.com\/ag-ui-protocol)\/\S+/i;
+
+export const RULES = [
+    'grounded-identifiers',
+    'source-link-or-handoff',
+    'handoff-is-short',
+    'no-banned-phrases',
+    'no-hedged-names',
+    'no-dead-package',
+] as const;
+
+export type RuleId = (typeof RULES)[number];
+
+export interface RuleResult {
+    rule: RuleId;
+    passed: boolean;
+    /** Why it failed, naming the offending text. Empty when it passed. */
+    detail: string;
+}
+
+function countWords(text: string): number {
+    const trimmed = text.trim();
+    return trimmed ? trimmed.split(/\s+/).length : 0;
+}
+
+/**
+ * Run every rule against one reply. Always returns one result per rule, so a
+ * report can distinguish "passed" from "not evaluated".
+ *
+ * `sources` must be the results the reply was actually generated from — the
+ * grounding rule is a lookup against them, so passing a different set silently
+ * turns the strictest rule into a no-op.
+ */
+export function checkReply(reply: string, sources: SearchResult[]): RuleResult[] {
+    const words = countWords(reply);
+    const cites = CITATION_LINK.test(reply);
+
+    // A reply that cites nothing is only acceptable as a short handoff, so the
+    // two rules below are the two halves of that single sentence in the doc.
+    const isShortEnoughForHandoff = words <= HANDOFF_WORD_CAP;
+
+    const groundedness = assessGroundedness(reply, sources);
+    const banned = BANNED_PHRASES.filter(({ pattern }) => pattern.test(reply));
+    const hedged = HEDGED_NAME_PATTERNS.filter((pattern) => pattern.test(reply));
+
+    return [
+        {
+            rule: 'grounded-identifiers',
+            passed: groundedness.unsourcedIdentifiers.length === 0,
+            detail: groundedness.unsourcedIdentifiers.length
+                ? `names not present in any source: ${groundedness.unsourcedIdentifiers.join(', ')}`
+                : '',
+        },
+        {
+            rule: 'source-link-or-handoff',
+            passed: cites || isShortEnoughForHandoff,
+            detail:
+                cites || isShortEnoughForHandoff
+                    ? ''
+                    : `${words} words with no docs or repo link; a reply this long has to cite what it came from`,
+        },
+        {
+            rule: 'handoff-is-short',
+            // Only binds when there is nothing to cite. An answer that carries a
+            // source has earned its length.
+            passed: cites || isShortEnoughForHandoff,
+            detail:
+                cites || isShortEnoughForHandoff
+                    ? ''
+                    : `uncited reply is ${words} words, over the ${HANDOFF_WORD_CAP}-word handoff cap`,
+        },
+        {
+            rule: 'no-banned-phrases',
+            passed: banned.length === 0,
+            detail: banned.map(({ why }) => why).join('; '),
+        },
+        {
+            rule: 'no-hedged-names',
+            passed: hedged.length === 0,
+            detail: hedged.length ? 'hedges an API name, which means it is guessing' : '',
+        },
+        {
+            rule: 'no-dead-package',
+            passed: !DEAD_PACKAGE.test(reply),
+            detail: DEAD_PACKAGE.test(reply)
+                ? 'mentions @copilotkitnext, which merged into @copilotkit v2'
+                : '',
+        },
+    ];
+}
diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts
index 75857e14..a56337b4 100644
--- a/packages/outpost/ai/src/index.ts
+++ b/packages/outpost/ai/src/index.ts
@@ -47,4 +47,8 @@ export type {
     TopIssueInput,
     ScoredTopIssue,
 } from './front-door.js';
+export { checkReply, RULES, HANDOFF_WORD_CAP } from './eval/rules.js';
+export type { RuleId, RuleResult } from './eval/rules.js';
+export { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './eval/harness.js';
+export type { EvalCase, CaseScore, EvalReport } from './eval/harness.js';
 export * from './types.js';

From 56579dd7ce02dd0c300de7cf736846e0dae3cce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?=
 <66887028+NathanTarbert@users.noreply.github.com>
Date: Wed, 26 Aug 2026 08:31:24 -0400
Subject: [PATCH 2/3] fix(ai): stop the eval score rewarding silence
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses the review on #241. Both blockers were right, and both were the
same shape: the metric reporting success where there was none.

## The score was maximised by saying nothing

Every rule was a prohibition, so `checkReply('')` passed all six — and so did
'No.' and 'Escalating.'. In the live mode that matters most, an agent
regressing toward empty replies would have shown up as the score IMPROVING,
in the tool built to catch exactly that.

New `says-something` rule with MIN_REPLY_WORDS = 8, set against the doc's own
floor rather than picked: the shortest acceptable reply is a Route, defined as
"two sentences. What we confirmed, if anything, and that a human is picking it
up." The doc's reference handoff is 15 words, so 8 leaves headroom while still
rejecting a bare acknowledgement. A SILENT reply is not a short reply, it is
no reply, and is never scored.

## An empty case list reported all-green

`scoreCases([])` gave every rule {passed: 0, total: 0}, and formatReport
printed six `ok` lines because `passed === total`. A live run whose fixture
loading silently produced nothing rendered as a perfect score. Now refused
with a message that names the likely cause.

## Also from the same review

- **`source-link-or-handoff` and `handoff-is-short` were one rule wearing two
  names** — identical expressions, so they could never disagree. That
  presented five independent signals as six and double-counted every failure,
  including in the summary I wrote for case D. Collapsed into
  `cites-or-is-a-short-handoff`.
- **The migration answer was the one reply the rules forbade.** "Never mention
  @copilotkitnext" is right as a default and wrong as an absolute: someone
  importing from it needs to be told what to import instead. Naming it is now
  allowed when the reply also names a live `@copilotkit/` package, which is
  what makes it a migration instruction rather than a stray reference.
  Otherwise the reporter who most needs that answer is the only one who cannot
  get it.
- **Test fixtures were exported from the package entry point.** HISTORICAL_
  FAILURES and TARGET_SHAPE put reconstructed bad replies containing
  `useCopilotFabricatedRender` and `@copilotkitnext/react` into dist and the
  worker image, where they would surface in a bundle grep for the dead package
  name. Import them from './eval/harness.js' directly instead.
- **Case D's fixture was corrupted by operator precedence** — `.repeat(2)`
  bound to the last string literal only, so the reply ended with a stray
  duplicate sentence rather than the padding the comment described. It is now
  120 words, which is what makes it a fixture for the length rule.
- **The SHADOW_MODE pointer was wrong** in a comment whose whole job is to say
  where to look. The gate is `ai-response.ts:824`; `:178` is unrelated code.
  Verified before changing.

Verification: ai package 305 -> 318, full repo turbo run test 10/10,
typecheck clean. Three mutations, each killing exactly the tests that name it:
says-something always passing, scoring an empty set, and a flat dead-package
ban.

Not addressed here, because both are about promoting these rules to a linter
rather than about the scorer: a reply can launder an invented name inside a
docs URL and pass the citation rule, and a correct answer built from the
plain-text fallback (sourceUrl undefined on every result) cannot pass it at
all. Both belong with CPK-8078.

Refs CPK-8076
---
 packages/outpost/ai/src/eval/harness.test.ts |  9 +++
 packages/outpost/ai/src/eval/harness.ts      | 21 +++++-
 packages/outpost/ai/src/eval/rules.test.ts   | 67 ++++++++++++++---
 packages/outpost/ai/src/eval/rules.ts        | 76 +++++++++++++++-----
 packages/outpost/ai/src/index.ts             | 10 ++-
 5 files changed, 150 insertions(+), 33 deletions(-)

diff --git a/packages/outpost/ai/src/eval/harness.test.ts b/packages/outpost/ai/src/eval/harness.test.ts
index b9dd60f8..2a3e7553 100644
--- a/packages/outpost/ai/src/eval/harness.test.ts
+++ b/packages/outpost/ai/src/eval/harness.test.ts
@@ -69,6 +69,15 @@ describe('the target shape passes', () => {
     });
 });
 
+// A live run whose fixture loading silently produced nothing used to render as a
+// perfect score: every rule {passed: 0, total: 0}, so `passed === total` printed
+// six `ok` lines.
+describe('an empty case list', () => {
+    it('is refused rather than scored as clean', () => {
+        expect(() => scoreCases([])).toThrow(/no cases/i);
+    });
+});
+
 describe('formatReport', () => {
     it('leads with the clean count and names each failing rule', () => {
         const text = formatReport(scoreCases(HISTORICAL_FAILURES));
diff --git a/packages/outpost/ai/src/eval/harness.ts b/packages/outpost/ai/src/eval/harness.ts
index 12a117d0..983b2fb4 100644
--- a/packages/outpost/ai/src/eval/harness.ts
+++ b/packages/outpost/ai/src/eval/harness.ts
@@ -19,7 +19,7 @@
  *     offline against real threads. This is the one that answers whether quality
  *     moved, and it cannot be deterministic, so it must not gate a merge.
  *
- * `SHADOW_MODE` gates only the platform post-back (`ai-response.ts:178`) —
+ * `SHADOW_MODE` gates only the platform post-back (`ai-response.ts:824`) —
  * retrieval and generation run fully either way — so the live mode needs no new
  * safety machinery, just a caller that feeds real threads through
  * `AIPipeline.generateSupportResponse` and hands the text here.
@@ -59,6 +59,18 @@ export interface EvalReport {
 }
 
 export function scoreCases(cases: EvalCase[]): EvalReport {
+    // Refuses an empty set rather than reporting one as clean. With no cases,
+    // every rule scored `{passed: 0, total: 0}` and `formatReport` printed six
+    // `ok` lines because `passed === total` — so a live run whose fixture loading
+    // silently produced nothing rendered as a perfect score. Silence
+    // indistinguishable from success, in the tool built to detect exactly that.
+    if (cases.length === 0) {
+        throw new Error(
+            'scoreCases received no cases. An empty set cannot be scored — it would ' +
+                'report every rule as passing. Check that the fixtures actually loaded.',
+        );
+    }
+
     const scored: CaseScore[] = cases.map((c) => {
         const results = checkReply(c.reply, c.sources);
         return {
@@ -182,7 +194,12 @@ export const HISTORICAL_FAILURES: EvalCase[] = [
             'the peer dependency range looks too wide. ' +
             "Here is what I can't do from here: I cannot read the source or run the install " +
             'to confirm either point. In the future, please include the full lockfile diff so ' +
-            'this is easier to triage. The team will take it from here. '.repeat(2),
+            'this is easier to triage. The team will take it from here. ' +
+            // Padded deliberately so the reply clears the handoff cap, which is
+            // half of what case D is a fixture FOR. Previously `.repeat(2)` bound
+            // to the last literal only, so the reply ended with a stray duplicate
+            // sentence rather than the length the comment claimed.
+            'Let me know if any of that needs clarifying and someone will pick it up. '.repeat(3),
         sources: CHAT_DOCS,
         provenance: 'https://github.com/CopilotKit/CopilotKit/issues/6423',
     },
diff --git a/packages/outpost/ai/src/eval/rules.test.ts b/packages/outpost/ai/src/eval/rules.test.ts
index e33718cb..022805cf 100644
--- a/packages/outpost/ai/src/eval/rules.test.ts
+++ b/packages/outpost/ai/src/eval/rules.test.ts
@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { checkReply, RULES, HANDOFF_WORD_CAP } from './rules.js';
+import { checkReply, RULES, HANDOFF_WORD_CAP, MIN_REPLY_WORDS } from './rules.js';
 import type { SearchResult } from '../types.js';
 
 const source = (content: string, title = 'CopilotChat'): SearchResult => ({
@@ -61,51 +61,96 @@ describe('grounded-identifiers', () => {
 // can't, it becomes a two-sentence handoff instead." So the rule is a
 // disjunction, not a flat requirement — a short handoff is allowed to have no
 // link, and that is the whole point of it existing.
-describe('source-link-or-handoff', () => {
+describe('cites-or-is-a-short-handoff', () => {
     it('fails a long answer that cites nothing', () => {
         const wordy = 'You can configure this in several ways. '.repeat(12);
-        expect(broken(wordy)).toContain('source-link-or-handoff');
+        expect(broken(wordy)).toContain('cites-or-is-a-short-handoff');
     });
 
     it('passes a long answer that links the docs', () => {
         const wordy =
             'You can configure this in several ways. '.repeat(12) +
             ' https://docs.copilotkit.ai/guides/configuration';
-        expect(broken(wordy)).not.toContain('source-link-or-handoff');
+        expect(broken(wordy)).not.toContain('cites-or-is-a-short-handoff');
     });
 
     it('passes a long answer that links a repo file, since code is a real answer', () => {
         const wordy =
             'This is handled by the adapter. '.repeat(12) +
             ' https://github.com/CopilotKit/CopilotKit/blob/main/packages/runtime/src/agent.ts';
-        expect(broken(wordy)).not.toContain('source-link-or-handoff');
+        expect(broken(wordy)).not.toContain('cites-or-is-a-short-handoff');
     });
 
     it('passes a short handoff with no link at all', () => {
         expect(
             broken('Confirmed the manifest and lockfile skew. Routing this to the team.'),
-        ).not.toContain('source-link-or-handoff');
+        ).not.toContain('cites-or-is-a-short-handoff');
     });
 });
 
 // "Nothing found -> two sentences, done." A no-answer that runs to 400 words is
 // the single thing the doc says changes the feel of the product most.
-describe('handoff-is-short', () => {
+describe('the handoff cap', () => {
     it('fails a handoff that pads past the cap', () => {
         const padded =
             'A human will follow up here shortly. ' +
             'In the meantime here is some general background. '.repeat(20);
         const results = checkReply(padded, DOCS);
-        const handoff = results.find((r) => r.rule === 'handoff-is-short');
-        expect(handoff?.passed).toBe(false);
-        expect(handoff?.detail).toContain(String(HANDOFF_WORD_CAP));
+        const rule = results.find((r) => r.rule === 'cites-or-is-a-short-handoff');
+        expect(rule?.passed).toBe(false);
+        expect(rule?.detail).toContain(String(HANDOFF_WORD_CAP));
     });
 
     it('does not apply the cap to an answer that carries a source', () => {
         const long =
             'Use the `CopilotChat` component. '.repeat(30) +
             ' https://docs.copilotkit.ai/reference/components/chat/CopilotChat';
-        expect(broken(long)).not.toContain('handoff-is-short');
+        expect(broken(long)).not.toContain('cites-or-is-a-short-handoff');
+    });
+});
+
+// Every other rule is a prohibition, so without this one the score is maximised
+// by saying nothing — and an agent regressing toward empty replies would read as
+// the score improving.
+describe('says-something', () => {
+    it.each(['', '   ', 'No.', 'Escalating.', 'Routing this to the team.'])(
+        'fails %o, which is not a reply',
+        (reply) => {
+            expect(broken(reply, [])).toContain('says-something');
+        },
+    );
+
+    it('passes the shortest reply the doc actually endorses', () => {
+        // The reference handoff, quoted in the doc as the right answer for case D.
+        expect(
+            broken('Confirmed the manifest and lockfile skew. Routing this to the team — someone will follow up here.'),
+        ).toEqual([]);
+    });
+
+    it('reports the word count so a failure is actionable', () => {
+        const result = checkReply('No.', DOCS).find((r) => r.rule === 'says-something');
+        expect(result?.detail).toContain('1 words');
+        expect(MIN_REPLY_WORDS).toBeGreaterThan(1);
+    });
+});
+
+// "Never mention @copilotkitnext" is right as a default and wrong as an
+// absolute: the reporter importing from it needs to be told what to import
+// instead, and under a flat ban that reply is the one that cannot be given.
+describe('the migration answer', () => {
+    it('allows naming the dead package when the live one is named too', () => {
+        expect(
+            broken(
+                "You're importing from `@copilotkitnext/react`, which merged into " +
+                    '`@copilotkit/react-core` v2 — switch the import and the hook names carry over.',
+            ),
+        ).not.toContain('no-dead-package');
+    });
+
+    it('still fails a stray reference with no replacement named', () => {
+        expect(broken('Install `@copilotkitnext/react` first and then retry the build.')).toContain(
+            'no-dead-package',
+        );
     });
 });
 
diff --git a/packages/outpost/ai/src/eval/rules.ts b/packages/outpost/ai/src/eval/rules.ts
index 05ca8177..dec7f442 100644
--- a/packages/outpost/ai/src/eval/rules.ts
+++ b/packages/outpost/ai/src/eval/rules.ts
@@ -44,6 +44,26 @@ import type { SearchResult } from '../types.js';
  */
 export const HANDOFF_WORD_CAP = 60;
 
+/**
+ * Fewest words that can count as a reply at all.
+ *
+ * Every other rule here is a prohibition, so without this one the score is
+ * MAXIMISED by saying nothing: `checkReply('')` passed all of them, and so did
+ * `'No.'` and `'Escalating.'`. In the live mode that matters most — an agent
+ * regressing toward empty or near-empty replies would show up as the score
+ * IMPROVING, in the tool built to catch exactly that.
+ *
+ * Set against the doc's own floor rather than picked: the shortest acceptable
+ * reply is a Route, and a Route is *"two sentences. What we confirmed, if
+ * anything, and that a human is picking it up."* The reference handoff in the
+ * doc — "Confirmed the manifest/lockfile skew. Routing this to the team —
+ * someone will follow up here." — is 15 words, so 8 leaves real headroom while
+ * still rejecting a bare acknowledgement.
+ *
+ * A SILENT reply is not a short reply, it is no reply, and is never scored here.
+ */
+export const MIN_REPLY_WORDS = 8;
+
 /**
  * Phrases the reply may never contain, each traceable to a case in the doc.
  *
@@ -90,14 +110,31 @@ const HEDGED_NAME_PATTERNS: RegExp[] = [
  */
 const DEAD_PACKAGE = /@copilotkitnext\b/i;
 
+/**
+ * The one reply that legitimately names the retired package.
+ *
+ * "Never mention `@copilotkitnext`" is right as a default and wrong as an
+ * absolute: someone importing from it needs to be told what to import instead,
+ * and *"you're importing from `@copilotkitnext/react`, which merged into
+ * `@copilotkit/react-core` v2 — switch the import"* is the correct answer. Under
+ * a flat ban that reply fails, and in the linter it collapses into a handoff —
+ * so the one reporter who most needs the migration answer is the only one who
+ * cannot get it.
+ *
+ * The carve-out is narrow: naming the dead package is allowed only when the
+ * reply also names a live `@copilotkit/` package, which is what makes it a
+ * migration instruction rather than a stray reference.
+ */
+const LIVE_PACKAGE = /@copilotkit\/[a-z-]+/i;
+
 /** A link that constitutes a citation: a docs page or a file in the repo. */
 const CITATION_LINK =
     /https?:\/\/(?:[a-z0-9-]+\.)*(?:copilotkit\.ai|github\.com\/CopilotKit|github\.com\/ag-ui-protocol)\/\S+/i;
 
 export const RULES = [
+    'says-something',
     'grounded-identifiers',
-    'source-link-or-handoff',
-    'handoff-is-short',
+    'cites-or-is-a-short-handoff',
     'no-banned-phrases',
     'no-hedged-names',
     'no-dead-package',
@@ -138,6 +175,14 @@ export function checkReply(reply: string, sources: SearchResult[]): RuleResult[]
     const hedged = HEDGED_NAME_PATTERNS.filter((pattern) => pattern.test(reply));
 
     return [
+        {
+            rule: 'says-something',
+            passed: words >= MIN_REPLY_WORDS,
+            detail:
+                words >= MIN_REPLY_WORDS
+                    ? ''
+                    : `${words} words — too short to be a reply; the shortest acceptable one is a two-sentence handoff`,
+        },
         {
             rule: 'grounded-identifiers',
             passed: groundedness.unsourcedIdentifiers.length === 0,
@@ -146,22 +191,16 @@ export function checkReply(reply: string, sources: SearchResult[]): RuleResult[]
                 : '',
         },
         {
-            rule: 'source-link-or-handoff',
+            // One rule, not two. `source-link-or-handoff` and `handoff-is-short`
+            // evaluated the identical expression, so they could never disagree —
+            // which presented five independent signals as six and double-counted
+            // every failure in both the per-rule table and the report.
+            rule: 'cites-or-is-a-short-handoff',
             passed: cites || isShortEnoughForHandoff,
             detail:
                 cites || isShortEnoughForHandoff
                     ? ''
-                    : `${words} words with no docs or repo link; a reply this long has to cite what it came from`,
-        },
-        {
-            rule: 'handoff-is-short',
-            // Only binds when there is nothing to cite. An answer that carries a
-            // source has earned its length.
-            passed: cites || isShortEnoughForHandoff,
-            detail:
-                cites || isShortEnoughForHandoff
-                    ? ''
-                    : `uncited reply is ${words} words, over the ${HANDOFF_WORD_CAP}-word handoff cap`,
+                    : `${words} words with no docs or repo link, over the ${HANDOFF_WORD_CAP}-word handoff cap; a reply this long has to cite what it came from`,
         },
         {
             rule: 'no-banned-phrases',
@@ -175,10 +214,11 @@ export function checkReply(reply: string, sources: SearchResult[]): RuleResult[]
         },
         {
             rule: 'no-dead-package',
-            passed: !DEAD_PACKAGE.test(reply),
-            detail: DEAD_PACKAGE.test(reply)
-                ? 'mentions @copilotkitnext, which merged into @copilotkit v2'
-                : '',
+            passed: !DEAD_PACKAGE.test(reply) || LIVE_PACKAGE.test(reply),
+            detail:
+                DEAD_PACKAGE.test(reply) && !LIVE_PACKAGE.test(reply)
+                    ? 'mentions @copilotkitnext without naming the @copilotkit/ package that replaced it'
+                    : '',
         },
     ];
 }
diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts
index a56337b4..071cbcd8 100644
--- a/packages/outpost/ai/src/index.ts
+++ b/packages/outpost/ai/src/index.ts
@@ -47,8 +47,14 @@ export type {
     TopIssueInput,
     ScoredTopIssue,
 } from './front-door.js';
-export { checkReply, RULES, HANDOFF_WORD_CAP } from './eval/rules.js';
+// The rule set and the scorer are API — Phase 3's linter consumes them.
+// HISTORICAL_FAILURES and TARGET_SHAPE deliberately are NOT: they are test data,
+// and exporting them put reconstructed bad replies containing
+// `useCopilotFabricatedRender` and `@copilotkitnext/react` into dist and the
+// worker image, where they would surface in a bundle grep for the dead package.
+// Import them from './eval/harness.js' directly in tests and offline runners.
+export { checkReply, RULES, HANDOFF_WORD_CAP, MIN_REPLY_WORDS } from './eval/rules.js';
 export type { RuleId, RuleResult } from './eval/rules.js';
-export { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './eval/harness.js';
+export { scoreCases, formatReport } from './eval/harness.js';
 export type { EvalCase, CaseScore, EvalReport } from './eval/harness.js';
 export * from './types.js';

From 1ef00fde2a9a9f4e1bd8ed9b45a8ff42683505ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?=
 <66887028+NathanTarbert@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:06:57 -0400
Subject: [PATCH 3/3] style(ai): format the eval module with prettier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The four eval files were committed unformatted. Nothing here changes
behaviour — 318 tests unchanged — but the repo's prettier config is the
convention and a formatting drift in new files becomes diff noise on every
later change to them.

Worth noting why this was not caught earlier: CI's job is named "Lint,
Typecheck & Test" and runs no lint step, because ESLint 9 cannot read the
repo's .eslintrc.cjs and the flat-config migration is still open (#141).
ci.yml:314 documents that. There is also no prettier check anywhere in CI, so
formatting is currently unenforced end to end.
---
 packages/outpost/ai/src/eval/harness.test.ts |  7 +------
 packages/outpost/ai/src/eval/harness.ts      |  2 +-
 packages/outpost/ai/src/eval/rules.test.ts   |  6 ++++--
 packages/outpost/ai/src/eval/rules.ts        | 18 ++++++++++++++----
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/packages/outpost/ai/src/eval/harness.test.ts b/packages/outpost/ai/src/eval/harness.test.ts
index 2a3e7553..32d45eca 100644
--- a/packages/outpost/ai/src/eval/harness.test.ts
+++ b/packages/outpost/ai/src/eval/harness.test.ts
@@ -1,10 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import {
-    scoreCases,
-    formatReport,
-    HISTORICAL_FAILURES,
-    TARGET_SHAPE,
-} from './harness.js';
+import { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './harness.js';
 import { RULES } from './rules.js';
 
 describe('scoreCases', () => {
diff --git a/packages/outpost/ai/src/eval/harness.ts b/packages/outpost/ai/src/eval/harness.ts
index 983b2fb4..67556253 100644
--- a/packages/outpost/ai/src/eval/harness.ts
+++ b/packages/outpost/ai/src/eval/harness.ts
@@ -236,5 +236,5 @@ export const TARGET_SHAPE: EvalCase = {
         },
     ],
     provenance:
-        'Maintainer reply quoted in the Agent\'s Output Doc, from the case-a thread (2026-08-08)',
+        "Maintainer reply quoted in the Agent's Output Doc, from the case-a thread (2026-08-08)",
 };
diff --git a/packages/outpost/ai/src/eval/rules.test.ts b/packages/outpost/ai/src/eval/rules.test.ts
index 022805cf..a4f157c4 100644
--- a/packages/outpost/ai/src/eval/rules.test.ts
+++ b/packages/outpost/ai/src/eval/rules.test.ts
@@ -123,7 +123,9 @@ describe('says-something', () => {
     it('passes the shortest reply the doc actually endorses', () => {
         // The reference handoff, quoted in the doc as the right answer for case D.
         expect(
-            broken('Confirmed the manifest and lockfile skew. Routing this to the team — someone will follow up here.'),
+            broken(
+                'Confirmed the manifest and lockfile skew. Routing this to the team — someone will follow up here.',
+            ),
         ).toEqual([]);
     });
 
@@ -160,7 +162,7 @@ describe('the migration answer', () => {
 describe('no-banned-phrases', () => {
     it.each([
         'Great question! Use the `CopilotChat` component.',
-        "Thanks for this detailed report. Use the `CopilotChat` component.",
+        'Thanks for this detailed report. Use the `CopilotChat` component.',
         "Here is what I can't do from here: read the source.",
         "I can't see other people's replies in this thread.",
         'In the future, please include a minimal reproduction in your issue.',
diff --git a/packages/outpost/ai/src/eval/rules.ts b/packages/outpost/ai/src/eval/rules.ts
index dec7f442..86f1c501 100644
--- a/packages/outpost/ai/src/eval/rules.ts
+++ b/packages/outpost/ai/src/eval/rules.ts
@@ -75,16 +75,26 @@ export const MIN_REPLY_WORDS = 8;
 const BANNED_PHRASES: Array<{ pattern: RegExp; why: string }> = [
     // Case D's opener, and the doc's "no praise openers" rule.
     { pattern: /\bgreat question\b/i, why: 'praise opener' },
-    { pattern: /\bthanks for (?:this|the|your) (?:detailed |thorough |thoughtful )?report\b/i, why: 'praise opener' },
+    {
+        pattern: /\bthanks for (?:this|the|your) (?:detailed |thorough |thoughtful )?report\b/i,
+        why: 'praise opener',
+    },
     { pattern: /\bexcellent (?:question|report|catch)\b/i, why: 'praise opener' },
     // Case D's "What I can't do from here" section, and the rule against the
     // agent performing its own humility.
     { pattern: /\bwhat i (?:can'?t|cannot) do\b/i, why: 'self-commentary about its own limits' },
-    { pattern: /\bi (?:haven'?t|have not) read the source\b/i, why: 'self-commentary about its own limits' },
-    { pattern: /\bi (?:don'?t|do not) have access to\b/i, why: 'self-commentary about its own limits' },
+    {
+        pattern: /\bi (?:haven'?t|have not) read the source\b/i,
+        why: 'self-commentary about its own limits',
+    },
+    {
+        pattern: /\bi (?:don'?t|do not) have access to\b/i,
+        why: 'self-commentary about its own limits',
+    },
     // Case C: it claimed it could not read the thread. It can.
     {
-        pattern: /\bi (?:can'?t|cannot) see (?:other|the other|anyone)[^.]{0,40}\b(?:replies|messages|responses)\b/i,
+        pattern:
+            /\bi (?:can'?t|cannot) see (?:other|the other|anyone)[^.]{0,40}\b(?:replies|messages|responses)\b/i,
         why: 'false claim that it cannot see the thread',
     },
     // Case D again: coaching the reporter on how to file better issues.