@@ -46,6 +46,35 @@ EVAL_REPORT_PATH=/tmp/agent-tool-use.json bunx vitest run evals/agent-tool-use
 The suite is also picked up by the normal `bun run test` run, so a regression
 fails CI even without the dedicated command.
 
+## Run against a real model (live)
+
+The same scenarios can be replayed against a live model. This is opt-in and
+never runs in CI. DeepSeek is wired first; any OpenAI-compatible provider works
+through `createOpenAICompatLiveCompletion` in `live.ts`.
+
+```sh
+cd apps/sim
+DEEPSEEK_API_KEY=... bun run test:evals:live
+```
+
+Useful knobs:
+
+| Variable | Default | Meaning |
+| --- | --- | --- |
+| `EVAL_TRIALS` | `3` | Runs per scenario. Models are nondeterministic, so results are pass rates. |
+| `EVAL_MIN_PASS_RATE` | `0` | When > 0, fail a scenario below this pass rate (0–1). |
+| `EVAL_MODEL` | `deepseek-chat` | Model id sent to the provider. |
+| `EVAL_TIMEOUT_MS` | `180000` | Per-request timeout. |
+| `EVAL_REPORT_PATH` | `test-results/evals/agent-tool-use-live.json` | Report location. |
+
+Live runs relax exact assertions: `toolCallSequence` becomes an ordered
+subsequence, `successfulToolCalls` becomes a minimum, and scripted-only cases
+(malformed JSON, unknown tool) are skipped. A scenario-level `liveExpect`
+overrides the scripted expectation where a real model cannot reproduce it (for
+example, an exact retry count). The report is at
+`test-results/evals/agent-tool-use-live.{json,md}` with pass rates, average
+iterations, latency, and the failed check names.
+
 ## Add a case
 
 1. Open [`agent-tool-use/scenarios.ts`](./agent-tool-use/scenarios.ts) and add
 
@@ -0,0 +1,83 @@
+import { providersMock } from '@sim/testing/mocks/providers.mock'
+import { providersConversationHistoryMock } from '@sim/testing/mocks/providers-conversation-history.mock'
+import { providersUtilsMock } from '@sim/testing/mocks/providers-utils.mock'
+import { toolsMock } from '@sim/testing/mocks/tools.mock'
+import { afterAll, describe, expect, it, vi } from 'vitest'
+import { runScenario } from '@/evals/agent-tool-use/harness'
+import { createDeepSeekLiveCompletion } from '@/evals/agent-tool-use/live'
+import { writeLiveEvalReport } from '@/evals/agent-tool-use/report'
+import { AGENT_TOOL_USE_SCENARIOS } from '@/evals/agent-tool-use/scenarios'
+import type { AgentToolUseResult, LiveScenarioSummary } from '@/evals/agent-tool-use/types'
+
+vi.mock('@/providers/conversation-history', () => providersConversationHistoryMock)
+vi.mock('@/tools', () => toolsMock)
+vi.mock('@/providers/utils', () => providersUtilsMock)
+vi.mock('@/providers', () => providersMock)
+
+/**
+ * Live agent tool-use evals. Opt-in only:
+ *
+ *   EVAL_LIVE=1 DEEPSEEK_API_KEY=... \
+ *     bun run --cwd apps/sim test --mode live evals/agent-tool-use/agent-tool-use.live.test.ts
+ *
+ * Each scenario runs `EVAL_TRIALS` times (default 3) because a real model is
+ * nondeterministic. The report carries pass rates, not a single boolean. Set
+ * `EVAL_MIN_PASS_RATE` (0–1) to turn a pass-rate floor into a failing gate.
+ */
+const LIVE = process.env.EVAL_LIVE === '1' && Boolean(process.env.DEEPSEEK_API_KEY)
+const TRIALS = Number(process.env.EVAL_TRIALS ?? '3')
+const MIN_PASS_RATE = Number(process.env.EVAL_MIN_PASS_RATE ?? '0')
+const MODEL = process.env.EVAL_MODEL ?? 'deepseek-chat'
+const TIMEOUT_MS = Number(process.env.EVAL_TIMEOUT_MS ?? '180000')
+
+const liveScenarios = AGENT_TOOL_USE_SCENARIOS.filter((scenario) => !scenario.scriptedOnly)
+const summaries: LiveScenarioSummary[] = []
+
+afterAll(() => {
+  if (!LIVE) return
+  writeLiveEvalReport(
+    summaries,
+    process.env.EVAL_REPORT_PATH ?? 'test-results/evals/agent-tool-use-live.json'
+  )
+})
+
+describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
+  it.each(liveScenarios)(
+    '$id: $name',
+    async (scenario) => {
+      const completion = createDeepSeekLiveCompletion(MODEL)
+      const results: AgentToolUseResult[] = []
+
+      for (let trial = 0; trial < TRIALS; trial++) {
+        results.push(
+          await runScenario(scenario, {
+            completion,
+            mode: 'live',
+            model: MODEL,
+            providerName: 'DeepSeek',
+          })
+        )
+      }
+
+      const passed = results.filter((result) => result.passed).length
+      const passRate = results.length === 0 ? 0 : passed / results.length
+      summaries.push({
+        id: scenario.id,
+        name: scenario.name,
+        category: scenario.category,
+        trials: results.length,
+        passed,
+        passRate,
+        results,
+      })
+
+      if (MIN_PASS_RATE > 0) {
+        expect(
+          passRate,
+          `${scenario.id} passed ${passed}/${results.length} trials`
+        ).toBeGreaterThanOrEqual(MIN_PASS_RATE)
+      }
+    },
+    TIMEOUT_MS
+  )
+})
@@ -12,9 +12,11 @@ import {
 } from '@/providers/openai-compat/streaming-tool-loop'
 import type { AgentStreamEvent } from '@/providers/stream-events'
 import type { StreamingToolLoopComplete } from '@/providers/streaming-tool-loop-shared'
+import { adaptOpenAIChatToolSchema } from '@/providers/tool-schema-adapter'
 import type { ProviderToolConfig, TimeSegment } from '@/providers/types'
 import type { ToolResponse } from '@/tools/types'
 import type {
+  AgentToolUseExpectations,
   AgentToolUseResult,
   AgentToolUseScenario,
   EvalCheck,
@@ -49,6 +51,20 @@ interface ToolCallList {
   count: number
 }
 
+/** Scripted runs assert exact behavior; live runs assert outcomes across trials. */
+export type EvalRunMode = 'scripted' | 'live'
+
+/** Options for {@link runScenario}. */
+export interface RunScenarioOptions {
+  /** Model turns. Defaults to the scenario's scripted turns. */
+  completion?: OpenAICompatCreateCompletion
+  mode?: EvalRunMode
+  /** Model id sent to a live provider and recorded in the run. */
+  model?: string
+  /** Provider label used in loop diagnostics. */
+  providerName?: string
+}
+
 /** Raw call counts as a value the loop never reads; scenarios only assert on it. */
 const COMPLETION_USAGE = (): CompletionUsage => ({
   prompt_tokens: 10,
@@ -184,22 +200,34 @@ function matchesContent(content: string, expected: string | RegExp): boolean {
   return typeof expected === 'string' ? content.includes(expected) : expected.test(content)
 }
 
+function isOrderedSubsequence(actual: string[], expected: string[]): boolean {
+  let index = 0
+  for (const name of actual) {
+    if (name === expected[index]) index += 1
+  }
+  return index === expected.length
+}
+
 function score(
-  scenario: AgentToolUseScenario,
+  expected: AgentToolUseExpectations,
   toolCalls: CapturedToolCall[],
   finalContent: string,
   iterations: number,
-  error: unknown
+  error: unknown,
+  mode: EvalRunMode
 ): EvalCheck[] {
-  const expected = scenario.expect
   const actualSequence = toolCalls.map((call) => call.name)
   const checks: EvalCheck[] = []
 
   if (expected.toolCallSequence) {
+    const sequenceMatches =
+      mode === 'live'
+        ? isOrderedSubsequence(actualSequence, expected.toolCallSequence)
+        : sameSequence(actualSequence, expected.toolCallSequence)
     checks.push(
       check(
         'tool-call-sequence',
-        sameSequence(actualSequence, expected.toolCallSequence),
+        sequenceMatches,
         `expected [${expected.toolCallSequence.join(', ')}], got [${actualSequence.join(', ')}]`
       )
     )
@@ -249,17 +277,22 @@ function score(
 
   const successful = toolCalls.filter((call) => call.success).length
   if (expected.successfulToolCalls !== undefined) {
+    const successMatches =
+      mode === 'live'
+        ? successful >= expected.successfulToolCalls
+        : successful === expected.successfulToolCalls
     checks.push(
       check(
         'successful-tool-calls',
-        successful === expected.successfulToolCalls,
-        `expected ${expected.successfulToolCalls}, got ${successful}`
+        successMatches,
+        `expected ${mode === 'live' ? 'at least ' : ''}${expected.successfulToolCalls}, got ${successful}`
       )
     )
   }
 
   const errored = toolCalls.length - successful
-  if (expected.erroredToolCalls !== undefined) {
+  /** A live model chooses its own retry count, so an exact error count is scripted-only. */
+  if (expected.erroredToolCalls !== undefined && mode !== 'live') {
     checks.push(
       check(
         'errored-tool-calls',
@@ -283,9 +316,18 @@ function score(
 }
 
 /** Runs and scores one scenario. */
-export async function runScenario(scenario: AgentToolUseScenario): Promise<AgentToolUseResult> {
+export async function runScenario(
+  scenario: AgentToolUseScenario,
+  options: RunScenarioOptions = {}
+): Promise<AgentToolUseResult> {
+  const mode = options.mode ?? 'scripted'
+  const model = options.model ?? EVAL_MODEL
+  const providerName = options.providerName ?? EVAL_PROVIDER
+  const expected =
+    mode === 'live' ? { ...scenario.expect, ...scenario.liveExpect } : scenario.expect
   const toolNames = new Set(scenario.tools.map((tool) => tool.name))
   const resultQueues = buildResultQueues(scenario, toolNames)
+  const providerTools = toProviderTools(scenario.tools)
   const invocations: EvalToolInvocation[] = []
 
   providersMock.MAX_TOOL_ITERATIONS = MAX_TOOL_ITERATIONS
@@ -312,18 +354,22 @@ export async function runScenario(scenario: AgentToolUseScenario): Promise<Agent
   const startedAt = Date.now()
   try {
     const stream = createOpenAICompatStreamingToolLoopStream({
-      providerName: EVAL_PROVIDER,
+      providerName,
       request: {
-        model: EVAL_MODEL,
+        model,
         apiKey: 'eval-key',
         messages: [],
-        tools: toProviderTools(scenario.tools),
+        tools: providerTools,
+      },
+      basePayload: {
+        model,
+        tools: providerTools.map((tool) => adaptOpenAIChatToolSchema(tool)),
       },
-      basePayload: { model: EVAL_MODEL },
       messages: [{ role: 'user', content: scenario.userMessage }],
-      createStream: createScriptedCompletion(scenario),
+      createStream: options.completion ?? createScriptedCompletion(scenario),
       logger,
       timeSegments,
+      preserveAssistantReasoning: true,
       onComplete: (result) => {
         completed = result
       },
@@ -337,7 +383,7 @@ export async function runScenario(scenario: AgentToolUseScenario): Promise<Agent
   const toolCalls = ((completed?.toolCalls as ToolCallList | undefined)?.list ?? []).slice()
   const finalContent = completed?.content ?? ''
   const iterations = completed?.iterations ?? 0
-  const checks = score(scenario, toolCalls, finalContent, iterations, streamError)
+  const checks = score(expected, toolCalls, finalContent, iterations, streamError, mode)
   const successful = toolCalls.filter((call) => call.success).length
 
   return {
 
@@ -0,0 +1,61 @@
+import OpenAI from 'openai'
+import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
+
+/**
+ * Real-model transport for the eval harness. Any OpenAI-compatible provider
+ * (OpenAI, DeepSeek, Groq, OpenRouter, …) works by passing its `baseURL`.
+ *
+ * The scripted suite stays the CI gate; this exists so the same scenarios can
+ * be replayed against a live model on demand.
+ */
+export interface OpenAICompatLiveModelOptions {
+  apiKey: string
+  model: string
+  baseURL?: string
+  /** Per-request timeout in ms. Live model calls routinely exceed the default. */
+  timeoutMs?: number
+}
+
+const DEFAULT_TIMEOUT_MS = 120_000
+
+export function createOpenAICompatLiveCompletion(
+  options: OpenAICompatLiveModelOptions
+): OpenAICompatCreateCompletion {
+  const client = new OpenAI({
+    apiKey: options.apiKey,
+    ...(options.baseURL ? { baseURL: options.baseURL } : {}),
+    timeout: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+    maxRetries: 2,
+  })
+
+  return async (params, requestOptions) =>
+    client.chat.completions.create(
+      {
+        ...params,
+        model: options.model,
+        stream: true,
+        // OpenAI-compatible providers require an opt-in to emit usage on streams.
+        stream_options: { include_usage: true },
+      },
+      requestOptions
+    )
+}
+
+/**
+ * DeepSeek's OpenAI-compatible endpoint. Reads `DEEPSEEK_API_KEY` and the
+ * optional `DEEPSEEK_BASE_URL` / `EVAL_MODEL` overrides.
+ */
+export function createDeepSeekLiveCompletion(
+  model = process.env.EVAL_MODEL ?? 'deepseek-chat'
+): OpenAICompatCreateCompletion {
+  const apiKey = process.env.DEEPSEEK_API_KEY
+  if (!apiKey) {
+    throw new Error('DEEPSEEK_API_KEY is required for the live agent eval')
+  }
+  return createOpenAICompatLiveCompletion({
+    apiKey,
+    baseURL: process.env.DEEPSEEK_BASE_URL ?? 'https://api.deepseek.com',
+    model,
+    ...(process.env.EVAL_TIMEOUT_MS ? { timeoutMs: Number(process.env.EVAL_TIMEOUT_MS) } : {}),
+  })
+}