|
| 1 | +import { existsSync, mkdtempSync, rmSync } from 'node:fs' |
| 2 | +import { tmpdir } from 'node:os' |
| 3 | +import { join } from 'node:path' |
| 4 | +import type { ChatCompletionChunk } from 'openai/resources/chat/completions' |
| 5 | +import { describe, expect, it } from 'vitest' |
| 6 | +import { |
| 7 | + createRecordingCompletion, |
| 8 | + createReplayCompletion, |
| 9 | + listReplayFixtures, |
| 10 | + replayFixturePath, |
| 11 | + writeReplayFixture, |
| 12 | +} from '@/evals/agent-tool-use/replay' |
| 13 | +import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop' |
| 14 | + |
| 15 | +function chunk(text: string): ChatCompletionChunk { |
| 16 | + return { |
| 17 | + id: 'chunk', |
| 18 | + object: 'chat.completion.chunk', |
| 19 | + created: 0, |
| 20 | + model: 'test', |
| 21 | + choices: [{ index: 0, delta: { content: text }, finish_reason: 'stop', logprobs: null }], |
| 22 | + } |
| 23 | +} |
| 24 | + |
| 25 | +/** A completion that yields each turn's chunks, one turn per call. */ |
| 26 | +function stagedCompletion(turns: ChatCompletionChunk[][]): OpenAICompatCreateCompletion { |
| 27 | + let index = 0 |
| 28 | + return async () => { |
| 29 | + const turn = turns[index] |
| 30 | + index += 1 |
| 31 | + return (async function* () { |
| 32 | + for (const item of turn) yield item |
| 33 | + })() |
| 34 | + } |
| 35 | +} |
| 36 | + |
| 37 | +async function drain(iterable: AsyncIterable<ChatCompletionChunk>): Promise<ChatCompletionChunk[]> { |
| 38 | + const seen: ChatCompletionChunk[] = [] |
| 39 | + for await (const item of iterable) seen.push(item) |
| 40 | + return seen |
| 41 | +} |
| 42 | + |
| 43 | +describe('replay primitives', () => { |
| 44 | + it('record then replay preserves every chunk of every turn', async () => { |
| 45 | + const original = [[chunk('a'), chunk('b')], [chunk('c')]] |
| 46 | + const recorded: ChatCompletionChunk[][] = [] |
| 47 | + const recording = createRecordingCompletion(stagedCompletion(original), (turn) => |
| 48 | + recorded.push(turn) |
| 49 | + ) |
| 50 | + |
| 51 | + const first = await drain(await recording({} as never)) |
| 52 | + const second = await drain(await recording({} as never)) |
| 53 | + |
| 54 | + expect(first).toEqual(original[0]) |
| 55 | + expect(second).toEqual(original[1]) |
| 56 | + expect(recorded).toEqual(original) |
| 57 | + |
| 58 | + const replay = createReplayCompletion(recorded) |
| 59 | + expect(await drain(await replay({} as never))).toEqual(original[0]) |
| 60 | + expect(await drain(await replay({} as never))).toEqual(original[1]) |
| 61 | + }) |
| 62 | + |
| 63 | + it('rejects when the fixture is exhausted', async () => { |
| 64 | + const replay = createReplayCompletion([[chunk('only')]]) |
| 65 | + await drain(await replay({} as never)) |
| 66 | + await expect(async () => replay({} as never)).rejects.toThrow( |
| 67 | + 'Replay fixture exhausted after 1 model turns' |
| 68 | + ) |
| 69 | + }) |
| 70 | + |
| 71 | + it('writes and lists fixtures from a directory', () => { |
| 72 | + const directory = mkdtempSync(join(tmpdir(), 'eval-replay-')) |
| 73 | + try { |
| 74 | + writeReplayFixture(directory, { |
| 75 | + scenarioId: 'single-tool-lookup', |
| 76 | + model: 'deepseek-chat', |
| 77 | + recordedAt: '2026-01-01T00:00:00.000Z', |
| 78 | + turns: [[chunk('a')]], |
| 79 | + }) |
| 80 | + |
| 81 | + expect(existsSync(replayFixturePath(directory, 'single-tool-lookup'))).toBe(true) |
| 82 | + const fixtures = listReplayFixtures(directory) |
| 83 | + expect(fixtures).toHaveLength(1) |
| 84 | + expect(fixtures[0]).toMatchObject({ |
| 85 | + scenarioId: 'single-tool-lookup', |
| 86 | + model: 'deepseek-chat', |
| 87 | + }) |
| 88 | + expect(fixtures[0].turns).toEqual([[chunk('a')]]) |
| 89 | + } finally { |
| 90 | + rmSync(directory, { recursive: true, force: true }) |
| 91 | + } |
| 92 | + }) |
| 93 | + |
| 94 | + it('returns no fixtures for a missing directory', () => { |
| 95 | + expect(listReplayFixtures(join(tmpdir(), 'eval-replay-does-not-exist'))).toEqual([]) |
| 96 | + }) |
| 97 | +}) |
0 commit comments