feat(evals): record and replay judge transcripts

Fixtures now store the judge model and judge turns alongside the task turns,
so replay reproduces a judged pass/fail instead of skipping the judge. A live
trial that failed only its judge check can no longer replay green.

- replay.ts: ReplayFixture gains judgeModel/judgeTurns
- live test records judge turns when EVAL_RECORD=1
- replay test supplies the recorded judge when the scenario has a rubric
- replay.test.ts round-trips judge turns (verified: a failing recorded verdict
  fails the replay)

Author: sudoKrishna

apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts

@@ -62,15 +62,20 @@ describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
     '$id: $name',
     async (scenario) => {
       const base = createDeepSeekLiveCompletion(MODEL)
-      const judgeCompletion = scenario.judge ? createDeepSeekLiveCompletion(JUDGE_MODEL) : undefined
+      const judgeBase = scenario.judge ? createDeepSeekLiveCompletion(JUDGE_MODEL) : undefined
       const results: AgentToolUseResult[] = []
 
       for (let trial = 0; trial < TRIALS; trial++) {
         const recordedTurns: ChatCompletionChunk[][] = []
+        const recordedJudgeTurns: ChatCompletionChunk[][] = []
         const completion =
           RECORD && trial === 0
             ? createRecordingCompletion(base, (turn) => recordedTurns.push(turn))
             : base
+        const judgeCompletion =
+          judgeBase && RECORD && trial === 0
+            ? createRecordingCompletion(judgeBase, (turn) => recordedJudgeTurns.push(turn))
+            : judgeBase
 
         results.push(
           await runScenario(scenario, {
@@ -96,6 +101,7 @@ describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
             model: MODEL,
             recordedAt: new Date().toISOString(),
             turns: recordedTurns,
+            ...(judgeBase ? { judgeModel: JUDGE_MODEL, judgeTurns: recordedJudgeTurns } : {}),
           })
         }
       }

apps/sim/evals/agent-tool-use/agent-tool-use.replay.test.ts

@@ -42,6 +42,15 @@ describe.skipIf(replayCases.length === 0)('agent tool-use replay suite', () => {
       mode: 'live',
       model: fixture.model,
       providerName: 'Replay',
+      ...(scenario.judge && fixture.judgeTurns
+        ? {
+            judge: {
+              completion: createReplayCompletion(fixture.judgeTurns),
+              model: fixture.judgeModel ?? 'replay-judge',
+              rubric: scenario.judge,
+            },
+          }
+        : {}),
     })
 
     const failed = result.checks.filter((entry) => !entry.passed)

apps/sim/evals/agent-tool-use/replay.test.ts

@@ -76,6 +76,8 @@ describe('replay primitives', () => {
         model: 'deepseek-chat',
         recordedAt: '2026-01-01T00:00:00.000Z',
         turns: [[chunk('a')]],
+        judgeModel: 'deepseek-chat',
+        judgeTurns: [[chunk('judge')]],
       })
 
       expect(existsSync(replayFixturePath(directory, 'single-tool-lookup'))).toBe(true)
@@ -84,8 +86,10 @@ describe('replay primitives', () => {
       expect(fixtures[0]).toMatchObject({
         scenarioId: 'single-tool-lookup',
         model: 'deepseek-chat',
+        judgeModel: 'deepseek-chat',
       })
       expect(fixtures[0].turns).toEqual([[chunk('a')]])
+      expect(fixtures[0].judgeTurns).toEqual([[chunk('judge')]])
     } finally {
       rmSync(directory, { recursive: true, force: true })
     }

apps/sim/evals/agent-tool-use/replay.ts

@@ -18,6 +18,10 @@ export interface ReplayFixture {
   recordedAt: string
   /** One entry per model call; each holds the chunks that call streamed. */
   turns: ChatCompletionChunk[][]
+  /** Judge model, present when the scenario was judged during recording. */
+  judgeModel?: string
+  /** Judge calls, so replay reproduces a judged pass/fail instead of skipping it. */
+  judgeTurns?: ChatCompletionChunk[][]
 }
 
 export function replayFixturePath(directory: string, scenarioId: string): string {