From 7fa07b5fa9575f1301e4a94834b8724c211ed1e8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 12 Aug 2026 22:40:46 -0600 Subject: [PATCH] fix(runtime): record served optimizer receipt model --- CHANGELOG.md | 5 ++ docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 2 +- package.json | 2 +- src/runtime/profile-chat-client.test.ts | 47 +++++++++++++++++-- src/runtime/profile-chat-client.ts | 19 ++++---- .../fixtures/agent-improvement-proposal.json | 10 ++-- .../agent-profile-improvement-proposal.json | 6 +-- 8 files changed, 69 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff981c07..53bc14aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.133.3 + +- Record the provider-served model in successful profile optimizer cost receipts, including routed snapshots. +- Keep declared model and unknown usage on failed or unproven execution paths. + ## 0.133.2 - Accept a routed response when its served-provider prefix differs from the profile's gateway prefix but its model name matches. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 01ac7ca3..8e880b0b 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.133.2` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.133.3` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 6fdbd53e..0fa5cf22 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.133.2.** +> **Version 0.133.3.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.145.2 <0.146.0`. > `sandbox` must satisfy `>=0.21.1 <0.22.0`. diff --git a/package.json b/package.json index 22daa9aa..1fedf4d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.133.2", + "version": "0.133.3", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts index d57d7e0a..e53a0729 100644 --- a/src/runtime/profile-chat-client.test.ts +++ b/src/runtime/profile-chat-client.test.ts @@ -71,7 +71,7 @@ describe('profileChatClient exact Runtime adapter', () => { expect(response.durationMs).toBeGreaterThanOrEqual(0) }) - it('keeps a provider snapshot in response evidence while receipt uses the declared model', async () => { + it('uses the provider-served snapshot in the successful optimizer receipt', async () => { const responseModel = 'deepseek-v4-flash@fp_a18b46594c_prod0820_fp8_kvcache_20260402' const call = profileOptimizerModelCall({ profile, @@ -99,14 +99,14 @@ describe('profileChatClient exact Runtime adapter', () => { if (!result.succeeded) throw new Error(result.error) expect(result.response.model).toBe(responseModel) expect(result.receipt).toMatchObject({ - model: 'deepseek-v4-flash', + model: responseModel, inputTokens: 3, outputTokens: 2, }) expect(result.execution).toMatchObject({ model: responseModel }) }) - it('accepts a provider-qualified snapshot for the exact profile model', async () => { + it('uses the provider-qualified served model in the successful optimizer receipt', async () => { const responseModel = 'deepseek/deepseek-v4-flash@fp_a18b46594c_prod0820_fp8_kvcache_20260402' const call = profileOptimizerModelCall({ profile, @@ -134,13 +134,52 @@ describe('profileChatClient exact Runtime adapter', () => { if (!result.succeeded) throw new Error(result.error) expect(result.response.model).toBe(responseModel) expect(result.receipt).toMatchObject({ - model: 'deepseek-v4-flash', + model: responseModel, inputTokens: 4, outputTokens: 3, }) expect(result.execution).toMatchObject({ model: responseModel }) }) + it('keeps the declared model on a failed optimizer receipt after rejecting the served model', async () => { + const call = profileOptimizerModelCall({ + profile, + context: 'failed optimizer model identity test', + executor: { + backend: 'router', + routerBaseUrl: 'http://injected.invalid/v1', + routerKey: 'injected-transport', + complete: async () => ({ + model: 'some-other-model', + choices: [{ message: { content: 'untrusted response' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 3, completion_tokens: 2, cost: 0.001 }, + }), + }, + }) + + const result = await call({ + callId: 'failed-optimizer-model-identity-1', + request: { ...request, model: 'deepseek-v4-flash' }, + endpointFormat: 'chat-completions', + signal: new AbortController().signal, + }) + + expect(result.succeeded).toBe(false) + if (result.succeeded) throw new Error('expected served-model validation failure') + expect(result.receipt).toMatchObject({ + model: 'deepseek-v4-flash', + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + costUnknown: true, + }) + expect(result.execution).toMatchObject({ + executed: true, + succeeded: false, + model: null, + }) + }) + it('carries the exact profile retry policy through the injected Router transport', async () => { let attempts = 0 const complete = vi.fn(async () => { diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts index 6ad3135f..0299018c 100644 --- a/src/runtime/profile-chat-client.ts +++ b/src/runtime/profile-chat-client.ts @@ -352,12 +352,11 @@ function optimizerReceipt( pricing: CustomTokenPricing | undefined, ): CostReceiptInput { const usage = run.turn.usage + const receiptModel = optimizerReceiptModel(model, run) const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd if (usage.tokensKnown === false) { return { - // Eval validates the receipt against the requested model. The observed model stays in - // the response and execution evidence for provider identity auditing. - model, + model: receiptModel, inputTokens: 0, outputTokens: 0, usageUnknown: true, @@ -375,9 +374,7 @@ function optimizerReceipt( throw new Error('profile optimizer cache classes exceed total input tokens') } return { - // Eval validates the receipt against the requested model. The observed model stays in the - // response and execution evidence for provider identity auditing. - model, + model: receiptModel, inputTokens: usage.input - classified, outputTokens: usage.output, ...(cachedTokens !== undefined ? { cachedTokens } : {}), @@ -400,12 +397,11 @@ function rawOptimizerReceipt( pricing: CustomTokenPricing | undefined, ): CostReceiptInput { const usage = run.turn.usage + const receiptModel = optimizerReceiptModel(model, run) const tokensKnown = usage.tokensKnown !== false const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd return { - // Eval validates the receipt against the requested model. The observed model stays in the - // execution evidence for provider identity auditing. - model, + model: receiptModel, inputTokens: tokensKnown ? usage.input : 0, outputTokens: tokensKnown ? usage.output : 0, ...(tokensKnown ? {} : { usageUnknown: true }), @@ -420,6 +416,11 @@ function rawOptimizerReceipt( } } +/** Successful exact turns use the provider-served model; failed turns remain fail-closed. */ +function optimizerReceiptModel(model: string, run: ProfileChatRun): string { + return run.succeeded ? run.response.model : model +} + function optimizerExecution( profileDigest: string, requestDigest: string, diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 0eedf55d..e7f5b2f9 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:a8ae0d0c88c99befa8b7c5e36f323f1b51a74b8e5aedf85d682a3b354f8dea9e", + "digest": "sha256:9065cde5cf9510a96007f157ba087e7e4691753dc6cb8f66100ab33996ca59d2", "evaluation": { "decision": { "contributingChecks": [ @@ -4870,7 +4870,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.133.2" + "runtimeVersion": "0.133.3" }, "objectives": [ { @@ -4981,8 +4981,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:758f051afe74b3790e4ec707c1e72f4145a02f09ee650c2b57cedcb50cdab0e4", - "runId": "agent-runtime-0.133.2-proposal-fixture", + "recordDigest": "sha256:17199cdb4119694d34a47081d069c00e68741ae433d85c7ebbbb8521c2017c47", + "runId": "agent-runtime-0.133.3-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5009,5 +5009,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.133.2-proposal-fixture" + "runId": "agent-runtime-0.133.3-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 03599040..08b4adb4 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:1a6f4cf84dc282a1813f9192638e5c5d3d2542f75ad5140cd6e1ba1b1fa973b3", + "digest": "sha256:0bcf37ef930ae0baad839db63c992b32668eef8b096f6bb642a8225dd67d7410", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.133.2" + "runtimeVersion": "0.133.3" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:dcb950dbca0154fbecf02d65c1a4de7d6d8b76db7dd2fd1ab230207b66512e8e", + "recordDigest": "sha256:db630beac38bfb4ef2dd2b505c567275997e04ef36911ad3a2e3eeac13e19f3c", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }