feat(ai): lint a draft against the reply rules before it publishes

packages/outpost/ai/src/eval/harness.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './harness.js';
 import { RULES } from './rules.js';
+import type { SearchResult } from '../types.js';
 
 describe('scoreCases', () => {
     it('reports every rule for every case', () => {
@@ -38,6 +39,24 @@ describe('the documented failures are all caught', () => {
         expect(failed).toContain('no-banned-phrases');
     });
 
+    // Looked up by id rather than index: the cases above are positional, so
+    // inserting anywhere but the end silently reassigns which case each of those
+    // assertions is about.
+    it('catches case E for the praise opener and the self-positioning paragraph', () => {
+        const caseE = HISTORICAL_FAILURES.find(
+            (c) => c.id === 'case-e-mcp-headers-self-commentary',
+        );
+        if (!caseE) throw new Error('case-e fixture is missing');
+
+        const failed = scoreCases([caseE]).cases[0].failed;
+
+        // `failed` carries rule ids, not which phrase fired, so this pins the
+        // fixture rather than the pattern — the opener alone would satisfy it.
+        // Which phrases are caught is pinned in rules.test.ts, where reverting the
+        // widened pattern fails six cases.
+        expect(failed).toContain('no-banned-phrases');
+    });
+
     it('catches case B for the dead package and the invented name', () => {
         const failed = scoreCases([HISTORICAL_FAILURES[1]]).cases[0].failed;
         expect(failed).toContain('no-dead-package');
@@ -73,6 +92,37 @@ describe('an empty case list', () => {
     });
 });
 
+// A not-applicable rule carries `passed: false`, so an unfiltered failing-cases
+// block printed `n/a` for a rule and then listed it as a failure two lines later —
+// re-creating the double-counting in the human-readable output.
+describe('the report does not list an unevaluated rule as a failure', () => {
+    const urlLess: SearchResult[] = [
+        { title: 'CopilotChat', content: 'Use the `CopilotChat` component.', score: 0.9 },
+    ];
+    const report = () =>
+        formatReport(
+            scoreCases([
+                {
+                    id: 'c1',
+                    question: 'q',
+                    reply: 'Great question! ' + 'padding word '.repeat(30),
+                    sources: urlLess,
+                    provenance: 'test',
+                },
+            ]),
+        );
+
+    it('marks the citation rule n/a in the per-rule table', () => {
+        expect(report()).toContain('n/a  cites-or-is-a-short-handoff: 0/0');
+    });
+
+    it('does not repeat it under the failing cases', () => {
+        const failingBlock = report().split('Failing cases:')[1] ?? '';
+        expect(failingBlock).toContain('no-banned-phrases');
+        expect(failingBlock).not.toContain('cites-or-is-a-short-handoff');
+    });
+});
+
 describe('formatReport', () => {
     it('leads with the clean count and names each failing rule', () => {
         const text = formatReport(scoreCases(HISTORICAL_FAILURES));

packages/outpost/ai/src/eval/harness.ts

@@ -76,18 +76,26 @@ export function scoreCases(cases: EvalCase[]): EvalReport {
         return {
             id: c.id,
             results,
-            failed: results.filter((r) => !r.passed).map((r) => r.rule),
+            // A rule that could not be evaluated is not a failure.
+            failed: results.filter((r) => r.applicable && !r.passed).map((r) => r.rule),
         };
     });
 
+    // `total` counts only the cases where the rule could be evaluated, so a rule
+    // that was inapplicable everywhere reads as 0/0 rather than as a clean sweep.
     const perRule = Object.fromEntries(
-        RULES.map((rule) => [
-            rule,
-            {
-                passed: scored.filter((s) => !s.failed.includes(rule)).length,
-                total: scored.length,
-            },
-        ]),
+        RULES.map((rule) => {
+            const applicable = scored.filter(
+                (s) => s.results.find((r) => r.rule === rule)?.applicable,
+            );
+            return [
+                rule,
+                {
+                    passed: applicable.filter((s) => !s.failed.includes(rule)).length,
+                    total: applicable.length,
+                },
+            ];
+        }),
     ) as Record<RuleId, { passed: number; total: number }>;
 
     return {
@@ -103,14 +111,23 @@ export function formatReport(report: EvalReport): string {
     const lines = [`${report.cleanCases}/${report.totalCases} cases clean`, ''];
     for (const rule of RULES) {
         const { passed, total } = report.perRule[rule];
-        lines.push(`  ${passed === total ? 'ok  ' : 'FAIL'} ${rule}: ${passed}/${total}`);
+        // `n/a` rather than `ok` when nothing exercised the rule — `passed === total`
+        // is trivially true at 0/0, which is how an unevaluated rule used to read as
+        // a clean sweep.
+        const verdict = total === 0 ? 'n/a ' : passed === total ? 'ok  ' : 'FAIL';
+        lines.push(`  ${verdict} ${rule}: ${passed}/${total}`);
     }
     const dirty = report.cases.filter((c) => c.failed.length > 0);
     if (dirty.length) {
         lines.push('', 'Failing cases:');
         for (const c of dirty) {
             lines.push(`  ${c.id}`);
-            for (const r of c.results.filter((r) => !r.passed)) {
+            // Filtered on `applicable` as well: a not-applicable rule carries
+            // `passed: false`, so without this the report printed `n/a` for a rule
+            // two lines above and then listed it as a failure — re-creating in the
+            // human-readable output exactly the double-counting removed from
+            // `perRule`.
+            for (const r of c.results.filter((r) => r.applicable && !r.passed)) {
                 lines.push(`    - ${r.rule}: ${r.detail}`);
             }
         }
@@ -203,6 +220,34 @@ export const HISTORICAL_FAILURES: EvalCase[] = [
         sources: CHAT_DOCS,
         provenance: 'https://github.com/CopilotKit/CopilotKit/issues/6423',
     },
+    {
+        id: 'case-e-mcp-headers-self-commentary',
+        question:
+            'v2 MCP sse servers silently drop the headers auth config — the documented example sends no Authorization header',
+        // CopilotKit#6927, posted 2026-09-06T17:40Z, 23 seconds after the issue
+        // opened. The reporter had already done the work: a reproduction, the
+        // wire-level symptom and a proposed fix. The reply opened by praising the
+        // write-up, then spent a paragraph announcing what it had not done, then
+        // handed the question back to engineering.
+        //
+        // Kept as a fixture because it is the failure the narrower
+        // `read the source` pattern missed on the verb alone: only the praise
+        // opener fired, so the self-positioning paragraph — the part that makes
+        // the reply worse than silence — published intact.
+        reply:
+            '## Thanks for this detailed report\n\n' +
+            'This is an exceptionally thorough write-up — the reproduction output, the proposed ' +
+            'fix, and the note about test coverage are all exactly what the maintainers need to ' +
+            'evaluate this quickly.\n\n' +
+            "To be clear about my position: I haven't run this code or inspected the source, so " +
+            "I can't confirm the root cause or validate the fix independently. What I can say is " +
+            'that the behavior you are describing is consistent with the kind of mismatch that ' +
+            'can happen when a parameter type changes shape across SDK versions.\n\n' +
+            'Engineering will need to verify the internal behavior and decide on the right fix.',
+        sources: [],
+        provenance:
+            'https://github.com/CopilotKit/CopilotKit/issues/6927#issuecomment — posted 2026-09-06, before #241/#242 merged',
+    },
 ];
 
 /**

packages/outpost/ai/src/eval/linter.test.ts

@@ -0,0 +1,141 @@
+import { describe, it, expect } from 'vitest';
+import { lintDraft, describeVerdict } from './linter.js';
+import type { SearchResult } from '../types.js';
+
+const DOCS: SearchResult[] = [
+    {
+        title: 'CopilotChat',
+        content: 'Use the `CopilotChat` component with the `instructions` prop.',
+        score: 0.9,
+        sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat',
+    },
+];
+
+/** A reply that breaks nothing: grounded, cited, no banned phrasing. */
+const GOOD =
+    'Use the `CopilotChat` component with the `instructions` prop. ' +
+    'https://docs.copilotkit.ai/reference/components/chat/CopilotChat';
+
+/** Case D's shape: praise opener, self-commentary, no citation, over the cap. */
+const BAD =
+    'Great question! Here is what I cannot do from here: I cannot read the source. ' +
+    'To summarise what you have found, the versions disagree. '.repeat(6);
+
+describe('report mode', () => {
+    it('is the default, so wiring it in cannot change what publishes', () => {
+        expect(lintDraft(BAD, DOCS).mode).toBe('report');
+    });
+
+    // The whole point of the mode: learn what enforcement would do to real
+    // traffic before letting it withhold anything from a real person.
+    it('publishes a failing draft while recording that it would not have', () => {
+        const verdict = lintDraft(BAD, DOCS);
+
+        expect(verdict.publish).toBe(true);
+        expect(verdict.wouldCollapse).toBe(true);
+        expect(verdict.failed).toContain('no-banned-phrases');
+    });
+
+    it('publishes a clean draft and says nothing would have collapsed', () => {
+        const verdict = lintDraft(GOOD, DOCS);
+
+        expect(verdict.publish).toBe(true);
+        expect(verdict.wouldCollapse).toBe(false);
+        expect(verdict.failed).toEqual([]);
+    });
+});
+
+describe('enforce mode', () => {
+    it('withholds a failing draft', () => {
+        const verdict = lintDraft(BAD, DOCS, 'enforce');
+
+        expect(verdict.publish).toBe(false);
+        expect(verdict.wouldCollapse).toBe(true);
+    });
+
+    it('publishes a clean draft', () => {
+        expect(lintDraft(GOOD, DOCS, 'enforce').publish).toBe(true);
+    });
+
+    // A rule that could not be evaluated must never withhold an answer. The live
+    // case: Pathfinder's plain-text fallback returns results with no sourceUrl, so
+    // a correct answer built from it has nothing it could cite — enforcing a
+    // citation there would collapse every such answer.
+    it('does not withhold on a rule that could not be evaluated', () => {
+        const urlLess: SearchResult[] = [
+            { title: 'CopilotChat', content: 'Use the `CopilotChat` component.', score: 0.9 },
+        ];
+        const long = 'Use the `CopilotChat` component as documented. '.repeat(12);
+
+        const verdict = lintDraft(long, urlLess, 'enforce');
+
+        expect(verdict.failed).not.toContain('cites-or-is-a-short-handoff');
+        expect(verdict.publish).toBe(true);
+    });
+
+    it('reports every rule either way, so a log line can show the whole picture', () => {
+        expect(lintDraft(GOOD, DOCS, 'enforce').results).toHaveLength(6);
+    });
+});
+
+describe('the reasons', () => {
+    it('name the rule and the offending text', () => {
+        const verdict = lintDraft('Great question! ' + 'padding word '.repeat(10), DOCS);
+
+        expect(verdict.reasons.join(' ')).toContain('no-banned-phrases');
+        expect(verdict.reasons.join(' ')).toContain('praise opener');
+    });
+});
+
+describe('describeVerdict', () => {
+    it('distinguishes a report-mode near-miss from an enforced collapse', () => {
+        expect(describeVerdict(lintDraft(BAD, DOCS), 'ticket-1')).toContain(
+            'published anyway (report mode)',
+        );
+        expect(describeVerdict(lintDraft(BAD, DOCS, 'enforce'), 'ticket-1')).toContain(
+            'collapsed to a handoff',
+        );
+    });
+
+    it('stays quiet-but-informative on a clean draft', () => {
+        expect(describeVerdict(lintDraft(GOOD, DOCS), 'ticket-1')).toBe(
+            '[DraftLinter] ticket-1: clean (report)',
+        );
+    });
+});
+
+// The harness scores the doc's metric — zero invented API names — while the
+// pipeline's groundedness gate suppresses only at two. A linter that collapsed at
+// one would withhold answers production publishes.
+describe('the metric and the publish gate are not the same', () => {
+    const SRC: SearchResult[] = [
+        {
+            title: 'CopilotChat',
+            content: 'Use the `CopilotChat` component.',
+            score: 0.9,
+            sourceUrl: 'https://docs.copilotkit.ai/reference',
+        },
+    ];
+    const cite = ' https://docs.copilotkit.ai/reference';
+
+    it('fails the metric on one invented name but still publishes', () => {
+        const reply =
+            'Call `useCopilotFabricated()` with CopilotChat as documented in the reference guide.' +
+            cite;
+        const verdict = lintDraft(reply, SRC, 'enforce');
+
+        expect(verdict.results.find((r) => r.rule === 'grounded-identifiers')?.passed).toBe(false);
+        expect(verdict.publish).toBe(true);
+        expect(verdict.failed).not.toContain('grounded-identifiers');
+    });
+
+    it('withholds at the production threshold of two', () => {
+        const reply =
+            'Call `useCopilotFabricated()` and `<CopilotInvented />` as documented in the guide.' +
+            cite;
+        const verdict = lintDraft(reply, SRC, 'enforce');
+
+        expect(verdict.publish).toBe(false);
+        expect(verdict.failed).toContain('grounded-identifiers');
+    });
+});