From 187e2c54bdf201b47ea28808a90139314c7b4e6c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 12 Aug 2026 22:40:57 -0600 Subject: [PATCH] fix(runtime): align optimizer receipt with served model --- CHANGELOG.md | 4 ++++ docs/api/primitive-catalog.md | 2 +- docs/canonical-api.md | 2 +- package.json | 2 +- src/runtime/profile-chat-client.test.ts | 6 +++--- src/runtime/profile-chat-client.ts | 20 +++++++++++-------- .../fixtures/agent-improvement-proposal.json | 10 +++++----- .../agent-profile-improvement-proposal.json | 6 +++--- 8 files changed, 30 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff981c07..a6313598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.133.3 + +- Preserve the provider-served model snapshot in official optimizer cost receipts so Eval can match the response, receipt, and execution evidence. + ## 0.133.2 - Accept a routed response when its served-provider prefix differs from the profile's gateway prefix but its model name matches. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 01ac7ca3..8e880b0b 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.133.2` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.133.3` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 6fdbd53e..0fa5cf22 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.133.2.** +> **Version 0.133.3.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.145.2 <0.146.0`. > `sandbox` must satisfy `>=0.21.1 <0.22.0`. diff --git a/package.json b/package.json index 22daa9aa..1fedf4d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.133.2", + "version": "0.133.3", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/runtime/profile-chat-client.test.ts b/src/runtime/profile-chat-client.test.ts index d57d7e0a..fa84385d 100644 --- a/src/runtime/profile-chat-client.test.ts +++ b/src/runtime/profile-chat-client.test.ts @@ -71,7 +71,7 @@ describe('profileChatClient exact Runtime adapter', () => { expect(response.durationMs).toBeGreaterThanOrEqual(0) }) - it('keeps a provider snapshot in response evidence while receipt uses the declared model', async () => { + it('uses the same provider snapshot in the response and optimizer receipt', async () => { const responseModel = 'deepseek-v4-flash@fp_a18b46594c_prod0820_fp8_kvcache_20260402' const call = profileOptimizerModelCall({ profile, @@ -99,7 +99,7 @@ describe('profileChatClient exact Runtime adapter', () => { if (!result.succeeded) throw new Error(result.error) expect(result.response.model).toBe(responseModel) expect(result.receipt).toMatchObject({ - model: 'deepseek-v4-flash', + model: responseModel, inputTokens: 3, outputTokens: 2, }) @@ -134,7 +134,7 @@ describe('profileChatClient exact Runtime adapter', () => { if (!result.succeeded) throw new Error(result.error) expect(result.response.model).toBe(responseModel) expect(result.receipt).toMatchObject({ - model: 'deepseek-v4-flash', + model: responseModel, inputTokens: 4, outputTokens: 3, }) diff --git a/src/runtime/profile-chat-client.ts b/src/runtime/profile-chat-client.ts index 6ad3135f..30ebcd8c 100644 --- a/src/runtime/profile-chat-client.ts +++ b/src/runtime/profile-chat-client.ts @@ -96,8 +96,9 @@ export function profileOptimizerModelCall(args: { } } const execution = optimizerExecution(profileDigest, requestDigest, request, run) + const receiptModel = optimizerReceiptModel(binding.model, run) try { - const receipt = optimizerReceipt(binding.model, run, args.pricing) + const receipt = optimizerReceipt(receiptModel, run, args.pricing) return run.succeeded ? { succeeded: true, @@ -111,7 +112,7 @@ export function profileOptimizerModelCall(args: { return { succeeded: false, error: message, - receipt: rawOptimizerReceipt(binding.model, run, args.pricing), + receipt: rawOptimizerReceipt(receiptModel, run, args.pricing), execution: { ...execution, succeeded: false, @@ -355,8 +356,8 @@ function optimizerReceipt( const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd if (usage.tokensKnown === false) { return { - // Eval validates the receipt against the requested model. The observed model stays in - // the response and execution evidence for provider identity auditing. + // The receipt model is the served identity when Runtime observed one. Eval and the response + // must carry the same identity, while this branch still preserves unknown token usage. model, inputTokens: 0, outputTokens: 0, @@ -375,8 +376,6 @@ function optimizerReceipt( throw new Error('profile optimizer cache classes exceed total input tokens') } return { - // Eval validates the receipt against the requested model. The observed model stays in the - // response and execution evidence for provider identity auditing. model, inputTokens: usage.input - classified, outputTokens: usage.output, @@ -403,8 +402,6 @@ function rawOptimizerReceipt( const tokensKnown = usage.tokensKnown !== false const actualCostUsd = usage.usdKnown === false ? undefined : usage.costUsd return { - // Eval validates the receipt against the requested model. The observed model stays in the - // execution evidence for provider identity auditing. model, inputTokens: tokensKnown ? usage.input : 0, outputTokens: tokensKnown ? usage.output : 0, @@ -420,6 +417,13 @@ function rawOptimizerReceipt( } } +function optimizerReceiptModel(model: string, run: ProfileChatRun): string { + const observedModel = run.succeeded ? run.response.model : run.turn.usage.model + return observedModel !== undefined && observedModelMatchesDeclared(observedModel, model) + ? observedModel + : model +} + function optimizerExecution( profileDigest: string, requestDigest: string, diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 0eedf55d..e7f5b2f9 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:a8ae0d0c88c99befa8b7c5e36f323f1b51a74b8e5aedf85d682a3b354f8dea9e", + "digest": "sha256:9065cde5cf9510a96007f157ba087e7e4691753dc6cb8f66100ab33996ca59d2", "evaluation": { "decision": { "contributingChecks": [ @@ -4870,7 +4870,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.133.2" + "runtimeVersion": "0.133.3" }, "objectives": [ { @@ -4981,8 +4981,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:758f051afe74b3790e4ec707c1e72f4145a02f09ee650c2b57cedcb50cdab0e4", - "runId": "agent-runtime-0.133.2-proposal-fixture", + "recordDigest": "sha256:17199cdb4119694d34a47081d069c00e68741ae433d85c7ebbbb8521c2017c47", + "runId": "agent-runtime-0.133.3-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5009,5 +5009,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.133.2-proposal-fixture" + "runId": "agent-runtime-0.133.3-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 03599040..08b4adb4 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:1a6f4cf84dc282a1813f9192638e5c5d3d2542f75ad5140cd6e1ba1b1fa973b3", + "digest": "sha256:0bcf37ef930ae0baad839db63c992b32668eef8b096f6bb642a8225dd67d7410", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.133.2" + "runtimeVersion": "0.133.3" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:dcb950dbca0154fbecf02d65c1a4de7d6d8b76db7dd2fd1ab230207b66512e8e", + "recordDigest": "sha256:db630beac38bfb4ef2dd2b505c567275997e04ef36911ad3a2e3eeac13e19f3c", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }