feat(ai): lint a draft against the reply rules before it publishes

packages/outpost/ai/src/eval/harness.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { scoreCases, formatReport, HISTORICAL_FAILURES, TARGET_SHAPE } from './harness.js';
 import { RULES } from './rules.js';
+import type { SearchResult } from '../types.js';
 
 describe('scoreCases', () => {
     it('reports every rule for every case', () => {
@@ -73,6 +74,37 @@ describe('an empty case list', () => {
     });
 });
 
+// A not-applicable rule carries `passed: false`, so an unfiltered failing-cases
+// block printed `n/a` for a rule and then listed it as a failure two lines later —
+// re-creating the double-counting in the human-readable output.
+describe('the report does not list an unevaluated rule as a failure', () => {
+    const urlLess: SearchResult[] = [
+        { title: 'CopilotChat', content: 'Use the `CopilotChat` component.', score: 0.9 },
+    ];
+    const report = () =>
+        formatReport(
+            scoreCases([
+                {
+                    id: 'c1',
+                    question: 'q',
+                    reply: 'Great question! ' + 'padding word '.repeat(30),
+                    sources: urlLess,
+                    provenance: 'test',
+                },
+            ]),
+        );
+
+    it('marks the citation rule n/a in the per-rule table', () => {
+        expect(report()).toContain('n/a  cites-or-is-a-short-handoff: 0/0');
+    });
+
+    it('does not repeat it under the failing cases', () => {
+        const failingBlock = report().split('Failing cases:')[1] ?? '';
+        expect(failingBlock).toContain('no-banned-phrases');
+        expect(failingBlock).not.toContain('cites-or-is-a-short-handoff');
+    });
+});
+
 describe('formatReport', () => {
     it('leads with the clean count and names each failing rule', () => {
         const text = formatReport(scoreCases(HISTORICAL_FAILURES));

packages/outpost/ai/src/eval/harness.ts

@@ -76,18 +76,26 @@ export function scoreCases(cases: EvalCase[]): EvalReport {
         return {
             id: c.id,
             results,
-            failed: results.filter((r) => !r.passed).map((r) => r.rule),
+            // A rule that could not be evaluated is not a failure.
+            failed: results.filter((r) => r.applicable && !r.passed).map((r) => r.rule),
         };
     });
 
+    // `total` counts only the cases where the rule could be evaluated, so a rule
+    // that was inapplicable everywhere reads as 0/0 rather than as a clean sweep.
     const perRule = Object.fromEntries(
-        RULES.map((rule) => [
-            rule,
-            {
-                passed: scored.filter((s) => !s.failed.includes(rule)).length,
-                total: scored.length,
-            },
-        ]),
+        RULES.map((rule) => {
+            const applicable = scored.filter(
+                (s) => s.results.find((r) => r.rule === rule)?.applicable,
+            );
+            return [
+                rule,
+                {
+                    passed: applicable.filter((s) => !s.failed.includes(rule)).length,
+                    total: applicable.length,
+                },
+            ];
+        }),
     ) as Record<RuleId, { passed: number; total: number }>;
 
     return {
@@ -103,14 +111,23 @@ export function formatReport(report: EvalReport): string {
     const lines = [`${report.cleanCases}/${report.totalCases} cases clean`, ''];
     for (const rule of RULES) {
         const { passed, total } = report.perRule[rule];
-        lines.push(`  ${passed === total ? 'ok  ' : 'FAIL'} ${rule}: ${passed}/${total}`);
+        // `n/a` rather than `ok` when nothing exercised the rule — `passed === total`
+        // is trivially true at 0/0, which is how an unevaluated rule used to read as
+        // a clean sweep.
+        const verdict = total === 0 ? 'n/a ' : passed === total ? 'ok  ' : 'FAIL';
+        lines.push(`  ${verdict} ${rule}: ${passed}/${total}`);
     }
     const dirty = report.cases.filter((c) => c.failed.length > 0);
     if (dirty.length) {
         lines.push('', 'Failing cases:');
         for (const c of dirty) {
             lines.push(`  ${c.id}`);
-            for (const r of c.results.filter((r) => !r.passed)) {
+            // Filtered on `applicable` as well: a not-applicable rule carries
+            // `passed: false`, so without this the report printed `n/a` for a rule
+            // two lines above and then listed it as a failure — re-creating in the
+            // human-readable output exactly the double-counting removed from
+            // `perRule`.
+            for (const r of c.results.filter((r) => r.applicable && !r.passed)) {
                 lines.push(`    - ${r.rule}: ${r.detail}`);
             }
         }

packages/outpost/ai/src/eval/linter.test.ts

@@ -0,0 +1,141 @@
+import { describe, it, expect } from 'vitest';
+import { lintDraft, describeVerdict } from './linter.js';
+import type { SearchResult } from '../types.js';
+
+const DOCS: SearchResult[] = [
+    {
+        title: 'CopilotChat',
+        content: 'Use the `CopilotChat` component with the `instructions` prop.',
+        score: 0.9,
+        sourceUrl: 'https://docs.copilotkit.ai/reference/components/chat/CopilotChat',
+    },
+];
+
+/** A reply that breaks nothing: grounded, cited, no banned phrasing. */
+const GOOD =
+    'Use the `CopilotChat` component with the `instructions` prop. ' +
+    'https://docs.copilotkit.ai/reference/components/chat/CopilotChat';
+
+/** Case D's shape: praise opener, self-commentary, no citation, over the cap. */
+const BAD =
+    'Great question! Here is what I cannot do from here: I cannot read the source. ' +
+    'To summarise what you have found, the versions disagree. '.repeat(6);
+
+describe('report mode', () => {
+    it('is the default, so wiring it in cannot change what publishes', () => {
+        expect(lintDraft(BAD, DOCS).mode).toBe('report');
+    });
+
+    // The whole point of the mode: learn what enforcement would do to real
+    // traffic before letting it withhold anything from a real person.
+    it('publishes a failing draft while recording that it would not have', () => {
+        const verdict = lintDraft(BAD, DOCS);
+
+        expect(verdict.publish).toBe(true);
+        expect(verdict.wouldCollapse).toBe(true);
+        expect(verdict.failed).toContain('no-banned-phrases');
+    });
+
+    it('publishes a clean draft and says nothing would have collapsed', () => {
+        const verdict = lintDraft(GOOD, DOCS);
+
+        expect(verdict.publish).toBe(true);
+        expect(verdict.wouldCollapse).toBe(false);
+        expect(verdict.failed).toEqual([]);
+    });
+});
+
+describe('enforce mode', () => {
+    it('withholds a failing draft', () => {
+        const verdict = lintDraft(BAD, DOCS, 'enforce');
+
+        expect(verdict.publish).toBe(false);
+        expect(verdict.wouldCollapse).toBe(true);
+    });
+
+    it('publishes a clean draft', () => {
+        expect(lintDraft(GOOD, DOCS, 'enforce').publish).toBe(true);
+    });
+
+    // A rule that could not be evaluated must never withhold an answer. The live
+    // case: Pathfinder's plain-text fallback returns results with no sourceUrl, so
+    // a correct answer built from it has nothing it could cite — enforcing a
+    // citation there would collapse every such answer.
+    it('does not withhold on a rule that could not be evaluated', () => {
+        const urlLess: SearchResult[] = [
+            { title: 'CopilotChat', content: 'Use the `CopilotChat` component.', score: 0.9 },
+        ];
+        const long = 'Use the `CopilotChat` component as documented. '.repeat(12);
+
+        const verdict = lintDraft(long, urlLess, 'enforce');
+
+        expect(verdict.failed).not.toContain('cites-or-is-a-short-handoff');
+        expect(verdict.publish).toBe(true);
+    });
+
+    it('reports every rule either way, so a log line can show the whole picture', () => {
+        expect(lintDraft(GOOD, DOCS, 'enforce').results).toHaveLength(6);
+    });
+});
+
+describe('the reasons', () => {
+    it('name the rule and the offending text', () => {
+        const verdict = lintDraft('Great question! ' + 'padding word '.repeat(10), DOCS);
+
+        expect(verdict.reasons.join(' ')).toContain('no-banned-phrases');
+        expect(verdict.reasons.join(' ')).toContain('praise opener');
+    });
+});
+
+describe('describeVerdict', () => {
+    it('distinguishes a report-mode near-miss from an enforced collapse', () => {
+        expect(describeVerdict(lintDraft(BAD, DOCS), 'ticket-1')).toContain(
+            'published anyway (report mode)',
+        );
+        expect(describeVerdict(lintDraft(BAD, DOCS, 'enforce'), 'ticket-1')).toContain(
+            'collapsed to a handoff',
+        );
+    });
+
+    it('stays quiet-but-informative on a clean draft', () => {
+        expect(describeVerdict(lintDraft(GOOD, DOCS), 'ticket-1')).toBe(
+            '[DraftLinter] ticket-1: clean (report)',
+        );
+    });
+});
+
+// The harness scores the doc's metric — zero invented API names — while the
+// pipeline's groundedness gate suppresses only at two. A linter that collapsed at
+// one would withhold answers production publishes.
+describe('the metric and the publish gate are not the same', () => {
+    const SRC: SearchResult[] = [
+        {
+            title: 'CopilotChat',
+            content: 'Use the `CopilotChat` component.',
+            score: 0.9,
+            sourceUrl: 'https://docs.copilotkit.ai/reference',
+        },
+    ];
+    const cite = ' https://docs.copilotkit.ai/reference';
+
+    it('fails the metric on one invented name but still publishes', () => {
+        const reply =
+            'Call `useCopilotFabricated()` with CopilotChat as documented in the reference guide.' +
+            cite;
+        const verdict = lintDraft(reply, SRC, 'enforce');
+
+        expect(verdict.results.find((r) => r.rule === 'grounded-identifiers')?.passed).toBe(false);
+        expect(verdict.publish).toBe(true);
+        expect(verdict.failed).not.toContain('grounded-identifiers');
+    });
+
+    it('withholds at the production threshold of two', () => {
+        const reply =
+            'Call `useCopilotFabricated()` and `<CopilotInvented />` as documented in the guide.' +
+            cite;
+        const verdict = lintDraft(reply, SRC, 'enforce');
+
+        expect(verdict.publish).toBe(false);
+        expect(verdict.failed).toContain('grounded-identifiers');
+    });
+});

packages/outpost/ai/src/eval/linter.ts

@@ -0,0 +1,106 @@
+/**
+ * The draft linter — step 3 of "Fix the Agent's Output".
+ *
+ * The doc's flowchart hangs on one arrow: *"If the draft breaks a rule, it
+ * doesn't get cleaned up and posted — it collapses into the two-sentence
+ * version. A short honest reply is always the fallback."* This module is that
+ * arrow. It runs the same rules the eval harness scores with — deliberately the
+ * same module, so the thing measured and the thing enforced cannot drift — and
+ * returns a decision.
+ *
+ * ## It returns a verdict, not copy
+ *
+ * `lintDraft` never produces replacement text. The caller substitutes its own,
+ * and in the pipeline that is the existing `SUPPRESSED_RESPONSE_TEXT` used by the
+ * groundedness gate. That is deliberate: that copy already promises a human
+ * follow-up, and #231 records what happens when two layers each add their own
+ * promise — the reporter is told twice. One owner for user-facing copy, one
+ * promise.
+ *
+ * ## Report-only by default
+ *
+ * `mode` defaults to `'report'`, which computes the verdict and changes nothing.
+ * Enforcing means a rule that misfires withholds a correct answer from a real
+ * person — the same failure direction as the groundedness gate suppressing one,
+ * which is the bug #234 was filed for. So the sequence is: run in report mode,
+ * read what it would have collapsed against real traffic, then enforce once the
+ * false-positive rate is known rather than assumed.
+ *
+ * A rule that could not be evaluated never collapses a draft. See
+ * `RuleResult.applicable` — Pathfinder's plain-text fallback returns results with
+ * no `sourceUrl` at all, so a correct answer built from it has nothing it could
+ * cite, and enforcing a citation there would collapse every such answer.
+ */
+
+import { checkReply } from './rules.js';
+import type { RuleId, RuleResult } from './rules.js';
+import type { SearchResult } from '../types.js';
+
+export type LintMode = 'report' | 'enforce';
+
+export interface LintVerdict {
+    /**
+     * Whether the caller should publish the draft.
+     *
+     * Always true in `'report'` mode, whatever the rules found — reporting is for
+     * learning what enforcement would do, so it must not change behaviour.
+     */
+    publish: boolean;
+    /** True when at least one applicable rule failed, regardless of mode. */
+    wouldCollapse: boolean;
+    /** The applicable rules the draft broke, in rule order. */
+    failed: RuleId[];
+    /** One line per failure, naming the offending text. */
+    reasons: string[];
+    /** Every rule's result, for logging and for the eval harness. */
+    results: RuleResult[];
+    mode: LintMode;
+}
+
+/**
+ * Judge a draft against the reply rules.
+ *
+ * `sources` must be the results the draft was generated from — the citation and
+ * identifier rules are lookups against them, so passing a different set silently
+ * turns the two strictest rules into no-ops.
+ */
+export function lintDraft(
+    reply: string,
+    sources: SearchResult[],
+    mode: LintMode = 'report',
+): LintVerdict {
+    const results = checkReply(reply, sources);
+    // Gates on `blocksPublish`, not on `passed`. The harness scores the doc's
+    // metric — zero invented API names — while the pipeline's groundedness gate
+    // suppresses only at two, and a linter that collapsed at one would withhold
+    // answers production publishes. See RuleResult.blocksPublish.
+    const broken = results.filter((r) => r.applicable && r.blocksPublish);
+
+    return {
+        publish: mode === 'report' ? true : broken.length === 0,
+        wouldCollapse: broken.length > 0,
+        failed: broken.map((r) => r.rule),
+        reasons: broken.map((r) => `${r.rule}: ${r.detail}`),
+        results,
+        mode,
+    };
+}
+
+/**
+ * One log line describing what the linter decided, for report-mode runs.
+ *
+ * Report mode is only worth anything if somebody can read the outcome, and #197
+ * is why this returns a string for the caller to log rather than reaching for a
+ * logger itself: nothing in this package has adopted the structured logger, so a
+ * caller-owned `console` line is the honest option rather than inventing a
+ * second convention here.
+ */
+export function describeVerdict(verdict: LintVerdict, context: string): string {
+    if (!verdict.wouldCollapse) {
+        return `[DraftLinter] ${context}: clean (${verdict.mode})`;
+    }
+    const action = verdict.publish
+        ? 'would have collapsed to a handoff, published anyway (report mode)'
+        : 'collapsed to a handoff';
+    return `[DraftLinter] ${context}: ${action} — ${verdict.reasons.join(' | ')}`;
+}