diff --git a/CHANGELOG.md b/CHANGELOG.md index a6313598..5120d240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.133.6 + +- The direct protected model-grant port accepts an optional caller-declared `maxTotalTokens` cap across input and output tokens. +- Bind that cap into the exact reservation response and reject a final ledger that exceeds it or omits token usage. +- Signed execution plans in the Interface 0.47 cohort remain unchanged; only the direct protected model-grant port accepts this field. + ## 0.133.3 - Preserve the provider-served model snapshot in official optimizer cost receipts so Eval can match the response, receipt, and execution evidence. diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 52350848..fa0e1235 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.8.8 + +- Verify candidate execution against Runtime 0.133.6 and its terminal aggregate-token integrity result. + ## 0.8.7 - Consume Runtime 0.132.11, Eval 0.145.2, Interface 0.47.0, Knowledge 7.2.4, and Sandbox 0.21.1 as one compatible dependency set. diff --git a/bench/package.json b/bench/package.json index 29b9624b..7718f22f 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.8.7", + "version": "0.8.8", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index fedbc4df..93e2ebd8 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -512,6 +512,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= preparationId, grantDigest: `sha256:${'c'.repeat(64)}`, closed: true, + usageWithinLimits: true, calls: [], }), }, diff --git a/docs/api/candidate-execution.md b/docs/api/candidate-execution.md index 16644f29..67c43ac3 100644 --- a/docs/api/candidate-execution.md +++ b/docs/api/candidate-execution.md @@ -523,6 +523,12 @@ Re-exports [AgentCandidateProtectedModelSettlement](index.md#agentcandidateprote *** +### AgentCandidateProtectedModelSettlementCall + +Re-exports [AgentCandidateProtectedModelSettlementCall](index.md#agentcandidateprotectedmodelsettlementcall) + +*** + ### AgentCandidateProtectedRunCapture Re-exports [AgentCandidateProtectedRunCapture](index.md#agentcandidateprotectedruncapture) diff --git a/docs/api/index.md b/docs/api/index.md index ae5c2c54..4d607679 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -2608,7 +2608,7 @@ Evaluator service must expire and revoke this reservation at this epoch millisec > **enforcedLimits**: [`AgentCandidateModelLimits`](#agentcandidatemodellimits) -The gateway must stop calls before any one of these limits is exceeded. +The gateway must stop calls before any declared model limit is exceeded. ##### network @@ -2646,9 +2646,15 @@ Injected only into the trusted executor after all pre-launch checks pass. > **closed**: `true` +##### usageWithinLimits + +> **usageWithinLimits**: `boolean` + +Router's terminal integrity result. False must never become a receipt. + ##### calls -> **calls**: readonly `AgentCandidateModelSettlementCall`[] +> **calls**: readonly [`AgentCandidateProtectedModelSettlementCall`](#agentcandidateprotectedmodelsettlementcall)[] *** @@ -10597,10 +10603,32 @@ Secret-free response from the service's reservation endpoint. ### AgentCandidateModelLimits -> **AgentCandidateModelLimits** = `Pick`\<`AgentCandidateExecutionLimits`, `"maxModelCalls"` \| `"maxInputTokens"` \| `"maxOutputTokens"` \| `"maxCostUsd"`\> +> **AgentCandidateModelLimits** = `Pick`\<`AgentCandidateExecutionLimits`, `"maxModelCalls"` \| `"maxInputTokens"` \| `"maxOutputTokens"` \| `"maxCostUsd"`\> & `object` Limits mechanically enforced by the evaluator-owned model gateway. +#### Type Declaration + +##### maxTotalTokens? + +> `optional` **maxTotalTokens?**: `number` + +Optional caller-declared cap across input and output tokens. + +*** + +### AgentCandidateProtectedModelSettlementCall + +> **AgentCandidateProtectedModelSettlementCall** = `AgentCandidateModelSettlementCall` & `object` + +Protected-port wire call with the gateway's counted input total preserved. + +#### Type Declaration + +##### accountedInputTokens + +> **accountedInputTokens**: `number` + *** ### AgentCandidateExecutorTaskOutcomeCapture diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 1d6aba04..5605207f 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.133.5` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.133.6` and `@tangle-network/agent-eval@0.145.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 430 exports. +Import from `@tangle-network/agent-runtime` — 431 exports. | Symbol | Kind | Summary | |---|---|---| @@ -241,6 +241,7 @@ Import from `@tangle-network/agent-runtime` — 430 exports. | `AgentCandidateModelGrantRunReservationInput` | type | Reservation fields supplied by a caller before Runtime resolves the model. | | `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | +| `AgentCandidateProtectedModelSettlementCall` | type | Protected-port wire call with the gateway's counted input total preserved. | | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | | `AgenticGeneratorShotDisposition` | type | Worktree decision emitted before a completed shot is retried, accepted, or | | `AgenticGeneratorShotExecution` | type | Runtime's exact terminal turn plus its complete normalized event stream. | @@ -1275,7 +1276,7 @@ Import from `@tangle-network/agent-runtime/primeintellect` — 29 exports. ### Candidate execution — immutable prepare, run, grade, and receipt -Import from `@tangle-network/agent-runtime/candidate-execution` — 113 exports. +Import from `@tangle-network/agent-runtime/candidate-execution` — 114 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1351,6 +1352,7 @@ Import from `@tangle-network/agent-runtime/candidate-execution` — 113 exports. | `AgentCandidateModelGrantRunReservationInput` | type | Reservation fields supplied by a caller before Runtime resolves the model. | | `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | +| `AgentCandidateProtectedModelSettlementCall` | type | Protected-port wire call with the gateway's counted input total preserved. | | `PersistedTaskOutcomeEvidence` | type | Immutable evaluator evidence retained with a verified candidate task outcome. | | `VerifiedAgentCandidateTaskOutcome` | type | Branded task outcome that has survived independent evaluator verification. | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index da947905..638700ca 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.133.5.** +> **Version 0.133.6.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.145.2 <0.146.0`. > `sandbox` must satisfy `>=0.21.1 <0.22.0`. diff --git a/package.json b/package.json index f1e74ff0..2206c8d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.133.5", + "version": "0.133.6", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/candidate-execution/index.ts b/src/candidate-execution/index.ts index 2784a0be..b16426b4 100644 --- a/src/candidate-execution/index.ts +++ b/src/candidate-execution/index.ts @@ -119,6 +119,7 @@ export { type AgentCandidateProtectedModelActivation, type AgentCandidateProtectedModelReservation, type AgentCandidateProtectedModelSettlement, + type AgentCandidateProtectedModelSettlementCall, type AgentCandidateProtectedRunCapture, type AgentCandidateRepositoryPort, type AgentCandidateRunFinalization, diff --git a/src/candidate-execution/model-settlement.ts b/src/candidate-execution/model-settlement.ts index 99c70486..3debbd86 100644 --- a/src/candidate-execution/model-settlement.ts +++ b/src/candidate-execution/model-settlement.ts @@ -10,6 +10,7 @@ const USD_NANOS = 1_000_000_000 export interface SealedAgentCandidateModelSettlement { readonly value: AgentCandidateProtectedModelSettlement + /** `inputTokens` is the gateway's accounted input total. */ readonly usage: AgentCandidateFixedSpend } @@ -20,10 +21,13 @@ export function sealAgentCandidateModelSettlement( ): SealedAgentCandidateModelSettlement { assertExactObjectKeys( settlement, - ['preparationId', 'grantDigest', 'closed', 'calls'], + ['preparationId', 'grantDigest', 'closed', 'usageWithinLimits', 'calls'], 'model settlement', ) if (settlement.closed !== true) throw new Error('protected model grant is not closed') + if (settlement.usageWithinLimits !== true) { + throw new Error('protected model settlement reports usage outside frozen limits') + } if (settlement.grantDigest !== expected.grantDigest) { throw new Error('protected model settlement grant digest does not match the reservation') } @@ -36,7 +40,7 @@ export function sealAgentCandidateModelSettlement( const callIds = new Set() const spanIds = new Set() - let inputTokens = 0 + let accountedInputTokens = 0 let outputTokens = 0 let cachedInputTokens = 0 let reasoningTokens = 0 @@ -54,6 +58,7 @@ export function sealAgentCandidateModelSettlement( 'startedAtMs', 'endedAtMs', 'inputTokens', + 'accountedInputTokens', 'outputTokens', 'cachedInputTokens', 'reasoningTokens', @@ -87,6 +92,12 @@ export function sealAgentCandidateModelSettlement( throw new Error(`model settlement call ${index} ended before it started`) } assertCount(source.inputTokens, `model settlement call ${index} inputTokens`) + assertCount(source.accountedInputTokens, `model settlement call ${index} accountedInputTokens`) + if (source.accountedInputTokens < source.inputTokens) { + throw new Error( + `model settlement call ${index} accountedInputTokens cannot be less than inputTokens`, + ) + } assertCount(source.outputTokens, `model settlement call ${index} outputTokens`) assertCount(source.cachedInputTokens, `model settlement call ${index} cachedInputTokens`) cachedInputTokens = safeAdd( @@ -101,7 +112,11 @@ export function sealAgentCandidateModelSettlement( throw new Error(`model settlement call ${index} has an invalid cost provenance`) } if (source.costProvenance === 'estimated') costProvenance = 'estimated' - inputTokens = safeAdd(inputTokens, source.inputTokens, 'input token total') + accountedInputTokens = safeAdd( + accountedInputTokens, + source.accountedInputTokens, + 'accounted input token total', + ) outputTokens = safeAdd(outputTokens, source.outputTokens, 'output token total') costUsdNanos = safeAdd(costUsdNanos, source.costUsdNanos, 'cost total') return Object.freeze({ ...source }) @@ -109,7 +124,7 @@ export function sealAgentCandidateModelSettlement( const usage = Object.freeze({ costUsdNanos, - inputTokens, + inputTokens: accountedInputTokens, outputTokens, cachedInputTokens, reasoningTokens, @@ -121,6 +136,7 @@ export function sealAgentCandidateModelSettlement( preparationId: settlement.preparationId, grantDigest: settlement.grantDigest, closed: true as const, + usageWithinLimits: true as const, calls: Object.freeze(calls), }), usage, diff --git a/src/candidate-execution/outcome-evidence.ts b/src/candidate-execution/outcome-evidence.ts index 6e3d6fa9..9e40af05 100644 --- a/src/candidate-execution/outcome-evidence.ts +++ b/src/candidate-execution/outcome-evidence.ts @@ -91,6 +91,8 @@ export async function persistCandidateModelSettlementEvidence( grantDigest: settlement.value.grantDigest, closed: true as const, resolved: identity.resolvedModel, + // Interface 0.47 signed evidence has no protected-port proof fields. + // The direct port validates them first and carries accounted input in usage. calls: settlement.value.calls.map((call) => ({ callId: call.callId, generationId: call.generationId, diff --git a/src/candidate-execution/protected-model-port.ts b/src/candidate-execution/protected-model-port.ts index 5111a9c7..2f88d4c7 100644 --- a/src/candidate-execution/protected-model-port.ts +++ b/src/candidate-execution/protected-model-port.ts @@ -93,6 +93,10 @@ export function createProtectedAgentCandidateModelPort( const gatewayDomain = assertGatewayDomain(options.gatewayDomain) const activationEnvNames = exactEnvironmentNames(options.activationEnvNames) const reservations = new Map() + const expiredReservations = new Map< + string, + { state: RememberedReservation; retainedUntilMs: number } + >() const recentSettlements = new Map() return { @@ -103,7 +107,7 @@ export function createProtectedAgentCandidateModelPort( }, reserveGrant: async (input) => { - pruneExpiredState(reservations, recentSettlements, Date.now()) + pruneExpiredState(reservations, expiredReservations, recentSettlements, Date.now()) const request = immutableCandidateValue(input) validateReserveInput(request) const requestDigest = canonicalCandidateDigest(request) @@ -111,6 +115,8 @@ export function createProtectedAgentCandidateModelPort( if (recentSettlements.has(key)) { throw new Error('protected model reservation is already settled') } + const expired = expiredReservations.get(key) + if (expired) throw new Error('protected model reservation has expired') const previous = reservations.get(key) if (previous && previous.requestDigest !== requestDigest) { throw new Error('protected model reservation retry changed immutable input') @@ -150,11 +156,14 @@ export function createProtectedAgentCandidateModelPort( }, activateGrant: async (input) => { - pruneExpiredState(reservations, recentSettlements, Date.now()) + pruneExpiredState(reservations, expiredReservations, recentSettlements, Date.now()) const request = immutableCandidateValue(input) const key = reservationKey(request.executionId, request.preparationId) if (recentSettlements.has(key)) throw new Error('protected model grant is already settled') const state = reservations.get(key) + if (!state && expiredReservations.has(key)) { + throw new Error('protected model grant reservation has expired') + } if (!state) throw new Error('protected model grant was not reserved by this port') assertGrantIdentity(state, request) if (state.settlementDigest) throw new Error('protected model grant is already settled') @@ -189,10 +198,10 @@ export function createProtectedAgentCandidateModelPort( settleGrant: async (input) => { const now = Date.now() - pruneExpiredState(reservations, recentSettlements, now) + pruneExpiredState(reservations, expiredReservations, recentSettlements, now) const request = immutableCandidateValue(input) const key = reservationKey(request.executionId, request.preparationId) - const state = reservations.get(key) + const state = reservations.get(key) ?? expiredReservations.get(key)?.state if (state) assertGrantIdentity(state, request) const remembered = state ?? recentSettlements.get(key) if (remembered?.settlementReason && remembered.settlementReason !== request.reason) { @@ -219,6 +228,7 @@ export function createProtectedAgentCandidateModelPort( state.settlementReason = request.reason } reservations.delete(key) + expiredReservations.delete(key) rememberSettlement(recentSettlements, key, { settlementDigest, settlementReason: request.reason, @@ -240,7 +250,9 @@ function validateReserveInput(input: AgentCandidateModelGrantReserveInput): void ['maxModelCalls', input.limits.maxModelCalls], ['maxInputTokens', input.limits.maxInputTokens], ['maxOutputTokens', input.limits.maxOutputTokens], + ['maxTotalTokens', input.limits.maxTotalTokens], ] as const) { + if (value === undefined) continue if (!Number.isSafeInteger(value) || value < 0) { throw new Error(`protected model reservation ${name} must be a nonnegative safe integer`) } @@ -269,7 +281,7 @@ function validateReservation( } exactRecord( source.enforcedLimits, - ['maxModelCalls', 'maxInputTokens', 'maxOutputTokens', 'maxCostUsd'], + modelLimitKeys(expected.limits), 'protected model reservation enforced limits', ) if ( @@ -399,6 +411,24 @@ function assertWithinReservedLimits( throw new Error(`protected model settlement ${name} ${actual} exceeds reserved ${limit}`) } } + const maxTotalTokens = reservation.limits.maxTotalTokens + if (maxTotalTokens !== undefined) { + const totalTokens = usage.inputTokens + usage.outputTokens + if (!Number.isSafeInteger(totalTokens)) { + throw new Error('protected model settlement total token usage is incomplete') + } + if (totalTokens > maxTotalTokens) { + throw new Error( + `protected model settlement total tokens ${totalTokens} exceeds reserved ${maxTotalTokens}`, + ) + } + } +} + +function modelLimitKeys(limits: AgentCandidateModelLimits): readonly string[] { + return limits.maxTotalTokens === undefined + ? ['maxModelCalls', 'maxInputTokens', 'maxOutputTokens', 'maxCostUsd'] + : ['maxModelCalls', 'maxInputTokens', 'maxOutputTokens', 'maxTotalTokens', 'maxCostUsd'] } function exactEnvironmentNames(values: readonly string[]): readonly string[] { @@ -453,11 +483,21 @@ function reservationKey(executionId: string, preparationId: string): string { function pruneExpiredState( reservations: Map, + expiredReservations: Map, settlements: Map, now: number, ): void { for (const [key, value] of reservations) { - if (value.expiresAtMs <= now) reservations.delete(key) + if (value.expiresAtMs <= now) { + reservations.delete(key) + expiredReservations.set(key, { + state: value, + retainedUntilMs: now + RECOVERED_SETTLEMENT_RETENTION_MS, + }) + } + } + for (const [key, value] of expiredReservations) { + if (value.retainedUntilMs <= now) expiredReservations.delete(key) } for (const [key, value] of settlements) { if (value.expiresAtMs <= now) settlements.delete(key) diff --git a/src/candidate-execution/types.ts b/src/candidate-execution/types.ts index 6e06381c..621347e2 100644 --- a/src/candidate-execution/types.ts +++ b/src/candidate-execution/types.ts @@ -165,14 +165,17 @@ export interface AgentCandidateModelPort { export type AgentCandidateModelLimits = Pick< AgentCandidateExecutionLimits, 'maxModelCalls' | 'maxInputTokens' | 'maxOutputTokens' | 'maxCostUsd' -> +> & { + /** Optional caller-declared cap across input and output tokens. */ + maxTotalTokens?: number +} export interface AgentCandidateProtectedModelReservation { preparationId: string digest: Sha256Digest /** Evaluator service must expire and revoke this reservation at this epoch millisecond. */ expiresAtMs: number - /** The gateway must stop calls before any one of these limits is exceeded. */ + /** The gateway must stop calls before any declared model limit is exceeded. */ enforcedLimits: AgentCandidateModelLimits /** Exact public endpoint exception; every other candidate destination stays blocked. */ network: AgentCandidateModelAccessNetwork @@ -187,7 +190,14 @@ export interface AgentCandidateProtectedModelSettlement { preparationId: string grantDigest: Sha256Digest closed: true - calls: readonly AgentCandidateModelSettlementCall[] + /** Router's terminal integrity result. False must never become a receipt. */ + usageWithinLimits: boolean + calls: readonly AgentCandidateProtectedModelSettlementCall[] +} + +/** Protected-port wire call with the gateway's counted input total preserved. */ +export type AgentCandidateProtectedModelSettlementCall = AgentCandidateModelSettlementCall & { + accountedInputTokens: number } export interface AgentCandidateMemoryResetResult { diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index faa6783b..34321a2b 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:964f9ea611150d403dec73eef15b931ac89a49e43dbcd4ee41604907ea9d140d", + "digest": "sha256:f4b1f74326477910b5e45daab3c25a86b0f5274a41dc8c743f09871bda3ed90f", "evaluation": { "decision": { "contributingChecks": [ @@ -4870,7 +4870,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.133.5" + "runtimeVersion": "0.133.6" }, "objectives": [ { @@ -4981,8 +4981,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:a34231bfcd78779ac70170cf587c71166313bb98589190fea1401f129f27f3e6", - "runId": "agent-runtime-0.133.5-proposal-fixture", + "recordDigest": "sha256:9e0a8384c31187fa102b043d870d7496993910e1e7a655f7e49cea29dc572bdd", + "runId": "agent-runtime-0.133.6-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5009,5 +5009,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.133.5-proposal-fixture" + "runId": "agent-runtime-0.133.6-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 71e8dcb6..5f709f1d 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:0f065aa9302a9257b694b391470642316e14e02e52265a9781e88c61c57ce843", + "digest": "sha256:8c1032f7eaccda248160a55e5176598d28bee8b46d3cc0ad47442ca74b3a714a", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.133.5" + "runtimeVersion": "0.133.6" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:bd7274edbf730006aa4981e3647d5a26ef76e374edbfe71b9edb2b999f824973", + "recordDigest": "sha256:86adb09c9680559bbe3e7ac5f7ae62cd3105c3a2c469fbcb0fbdb177e5325937", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/candidate-execution-dispose.test.ts b/tests/candidate-execution-dispose.test.ts index 7a73be9a..fa55c182 100644 --- a/tests/candidate-execution-dispose.test.ts +++ b/tests/candidate-execution-dispose.test.ts @@ -23,6 +23,7 @@ describe('prepared candidate disposal', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], } } @@ -60,6 +61,7 @@ describe('prepared candidate disposal', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], } } @@ -90,6 +92,7 @@ describe('prepared candidate disposal', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], } } diff --git a/tests/candidate-execution-execute.test.ts b/tests/candidate-execution-execute.test.ts index 67ea8794..e745bfaa 100644 --- a/tests/candidate-execution-execute.test.ts +++ b/tests/candidate-execution-execute.test.ts @@ -365,6 +365,7 @@ describe('atomic prepared candidate execution', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [ { callId: 'call-paid-success', @@ -375,6 +376,7 @@ describe('atomic prepared candidate execution', () => { startedAtMs: 120, endedAtMs: 180, inputTokens: 10, + accountedInputTokens: 10, outputTokens: 5, cachedInputTokens: 2, reasoningTokens: 1, @@ -476,7 +478,13 @@ describe('atomic prepared candidate execution', () => { } fixture.ports.models.settleGrant = async ({ preparationId }) => { settlements++ - return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } const result = await executePreparedAgentCandidate( prepared, @@ -631,7 +639,13 @@ describe('atomic prepared candidate execution', () => { }) fixture.ports.models.settleGrant = async ({ preparationId }) => { advanceClock(cleanupTimeoutMs - 1) - return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), @@ -848,6 +862,7 @@ describe('atomic prepared candidate execution', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], } } @@ -1095,6 +1110,7 @@ describe('atomic prepared candidate execution', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], } } @@ -1171,6 +1187,7 @@ describe('atomic prepared candidate execution', () => { preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [ { callId: 'call-paid-1', @@ -1181,6 +1198,7 @@ describe('atomic prepared candidate execution', () => { startedAtMs: 100, endedAtMs: 150, inputTokens: 10, + accountedInputTokens: 10, outputTokens: 5, cachedInputTokens: 0, reasoningTokens: 0, @@ -1229,7 +1247,13 @@ describe('atomic prepared candidate execution', () => { let settledAfterStop = false fixture.ports.models.settleGrant = async ({ preparationId }) => { settledAfterStop = stoppedAfterAbort - return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } vi.useFakeTimers({ now: Date.now() }) const startedAt = Date.now() @@ -1370,7 +1394,13 @@ describe('atomic prepared candidate execution', () => { let disposals = 0 fixture.ports.models.settleGrant = async ({ preparationId }) => { settlements++ - return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), @@ -1563,7 +1593,13 @@ describe('atomic prepared candidate execution', () => { } fixture.ports.models.settleGrant = async ({ preparationId }) => { order.push('model-settle') - return { preparationId, grantDigest: candidateSha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: candidateSha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } const prepared = await prepareAgentCandidateExecution( await verifyAgentCandidateBundle(fixture.bundle, fixture.ports), diff --git a/tests/candidate-execution-finalize.test.ts b/tests/candidate-execution-finalize.test.ts index b5e48752..5a04ddea 100644 --- a/tests/candidate-execution-finalize.test.ts +++ b/tests/candidate-execution-finalize.test.ts @@ -71,6 +71,7 @@ async function finalizePrepared( preparationId: state.preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [ { callId: 'call-1', @@ -81,6 +82,7 @@ async function finalizePrepared( startedAtMs: 120, endedAtMs: 200, inputTokens: 10, + accountedInputTokens: 10, outputTokens: 5, cachedInputTokens: 2, reasoningTokens: 0, @@ -432,6 +434,7 @@ describe('protected candidate run finalization', () => { preparationId: assertPreparedCandidateIntegrity(execution).preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [ { callId: 'call-1', @@ -442,6 +445,7 @@ describe('protected candidate run finalization', () => { startedAtMs: 120, endedAtMs: 180, inputTokens: 5, + accountedInputTokens: 5, outputTokens: 2, cachedInputTokens: 0, reasoningTokens: 0, @@ -457,6 +461,7 @@ describe('protected candidate run finalization', () => { startedAtMs: 220, endedAtMs: 280, inputTokens: 5, + accountedInputTokens: 5, outputTokens: 2, cachedInputTokens: 0, reasoningTokens: 0, diff --git a/tests/candidate-execution-model-port.test.ts b/tests/candidate-execution-model-port.test.ts index 9f16f103..288ea3d4 100644 --- a/tests/candidate-execution-model-port.test.ts +++ b/tests/candidate-execution-model-port.test.ts @@ -1,10 +1,6 @@ import { InMemoryTraceStore, type LlmSpan } from '@tangle-network/agent-eval' -import type { - AgentCandidateModelSettlementCall, - AgentCandidateResolvedModel, - Sha256Digest, -} from '@tangle-network/agent-interface' -import { describe, expect, it } from 'vitest' +import type { AgentCandidateResolvedModel, Sha256Digest } from '@tangle-network/agent-interface' +import { describe, expect, it, vi } from 'vitest' import { appendAuthoritativeModelSettlementSpans, assertTraceMatchesModelSettlement, @@ -23,6 +19,7 @@ import type { AgentCandidateModelPort, AgentCandidateProtectedModelActivation, AgentCandidateProtectedModelSettlement, + AgentCandidateProtectedModelSettlementCall, } from '../src/candidate-execution/types' const GATEWAY_DOMAIN = 'router.tangle.tools' @@ -99,9 +96,9 @@ function settleInput( function modelCall( index: number, - overrides: Partial = {}, -): AgentCandidateModelSettlementCall { - return { + overrides: Partial = {}, +): AgentCandidateProtectedModelSettlementCall { + const base = { callId: `call-${index}`, generationId: `generation-${index}`, traceSpanId: `generation-${index}`, @@ -110,22 +107,29 @@ function modelCall( startedAtMs: 1_000 + index * 100, endedAtMs: 1_050 + index * 100, inputTokens: 10, + accountedInputTokens: 10, outputTokens: 5, cachedInputTokens: 0, reasoningTokens: 0, costUsdNanos: 10_000_000, costProvenance: 'observed' as const, + } + return { + ...base, ...overrides, + accountedInputTokens: + overrides.accountedInputTokens ?? overrides.inputTokens ?? base.inputTokens, } } function settlement( - calls: readonly AgentCandidateModelSettlementCall[] = [], + calls: readonly AgentCandidateProtectedModelSettlementCall[] = [], ): AgentCandidateProtectedModelSettlement { return { preparationId: 'preparation-1', grantDigest: sha('b'), closed: true, + usageWithinLimits: true, calls, } } @@ -494,6 +498,138 @@ describe('protected candidate model port', () => { ) }) + it('rejects aggregate token overflow when each channel remains below its own cap', async () => { + const limits = { + ...reserveInput().limits, + maxTotalTokens: 40, + } + const client = fakeClient({ + settle: async () => + settlement([ + modelCall(1, { + inputTokens: 30, + outputTokens: 20, + costUsdNanos: 0, + }), + ]), + }) + const port = createPort(client) + await reserve(port, reserveInput(limits)) + await expect(port.settleGrant(settleInput())).rejects.toThrow( + /total tokens 50 exceeds reserved 40/, + ) + }) + + it('accepts the exact aggregate token boundary and binds it to the reservation', async () => { + const limits = { + ...reserveInput().limits, + maxTotalTokens: 40, + } + const client = fakeClient({ + reserve: async (input) => reservation(input), + settle: async () => + settlement([ + modelCall(1, { + inputTokens: 25, + outputTokens: 15, + costUsdNanos: 0, + }), + ]), + }) + const port = createPort(client) + await expect(reserve(port, reserveInput(limits))).resolves.toMatchObject({ + enforcedLimits: limits, + }) + await expect(port.settleGrant(settleInput())).resolves.toEqual( + settlement([ + modelCall(1, { + inputTokens: 25, + outputTokens: 15, + costUsdNanos: 0, + }), + ]), + ) + }) + + it('counts the Router accounted input total for aggregate enforcement', async () => { + const limits = { + ...reserveInput().limits, + maxTotalTokens: 40, + } + const client = fakeClient({ + settle: async () => + settlement([ + modelCall(1, { + inputTokens: 20, + accountedInputTokens: 30, + outputTokens: 15, + costUsdNanos: 0, + }), + ]), + }) + const port = createPort(client) + await reserve(port, reserveInput(limits)) + await expect(port.settleGrant(settleInput())).rejects.toThrow( + /total tokens 45 exceeds reserved 40/, + ) + }) + + it('fails closed when Router reports a settlement outside frozen limits', async () => { + const client = fakeClient({ + settle: async () => ({ ...settlement(), usageWithinLimits: false }), + }) + const port = createPort(client) + await reserve(port) + await expect(port.settleGrant(settleInput())).rejects.toThrow( + /reports usage outside frozen limits/, + ) + }) + + it('keeps expired reservation limits for terminal local validation', async () => { + const limits = { + ...reserveInput().limits, + maxTotalTokens: 40, + } + const expiresAtMs = Date.now() + 10 + const client = fakeClient({ + settle: async () => + settlement([ + modelCall(1, { + inputTokens: 30, + outputTokens: 20, + costUsdNanos: 0, + }), + ]), + }) + const port = createPort(client) + await reserve(port, { ...reserveInput(limits), expiresAtMs }) + const clock = vi.spyOn(Date, 'now').mockReturnValue(expiresAtMs + 1) + try { + await expect(port.settleGrant(settleInput())).rejects.toThrow( + /total tokens 50 exceeds reserved 40/, + ) + } finally { + clock.mockRestore() + } + }) + + it('rejects incomplete token usage before claiming aggregate compliance', async () => { + const limits = { + ...reserveInput().limits, + maxTotalTokens: 40, + } + const partialCall = modelCall(1, { inputTokens: 25, outputTokens: 15, costUsdNanos: 0 }) + delete (partialCall as Partial).outputTokens + const client = fakeClient({ + settle: async () => settlement([partialCall as AgentCandidateProtectedModelSettlementCall]), + }) + const port = createPort(client) + await reserve(port, reserveInput(limits)) + await expect(port.settleGrant(settleInput())).rejects.toThrow( + /model settlement call 0 is missing field outputTokens/, + ) + }) + it.each([ [ 'duplicate call ids', diff --git a/tests/candidate-execution-prepare.test.ts b/tests/candidate-execution-prepare.test.ts index 08adf118..673b6b54 100644 --- a/tests/candidate-execution-prepare.test.ts +++ b/tests/candidate-execution-prepare.test.ts @@ -618,7 +618,13 @@ describe('candidate execution preparation', () => { }) unenforced.ports.models.settleGrant = async ({ preparationId }) => { settledReservations++ - return { preparationId, grantDigest: sha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: sha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } await expect( prepareAgentCandidateExecution( @@ -642,7 +648,13 @@ describe('candidate execution preparation', () => { }) invalid.ports.models.settleGrant = async ({ preparationId }) => { settledReservations++ - return { preparationId, grantDigest: sha('c'), closed: true, calls: [] } + return { + preparationId, + grantDigest: sha('c'), + closed: true, + usageWithinLimits: true, + calls: [], + } } await expect( prepareAgentCandidateExecution( @@ -704,6 +716,7 @@ describe('candidate execution preparation', () => { preparationId: `candidate-preparation.${'A'.repeat(43)}`, grantDigest: sha('c'), closed: true, + usageWithinLimits: true, calls: [], }) await expect( diff --git a/tests/exact-process-candidate.test.ts b/tests/exact-process-candidate.test.ts index a2fa9923..c6502929 100644 --- a/tests/exact-process-candidate.test.ts +++ b/tests/exact-process-candidate.test.ts @@ -95,6 +95,7 @@ describe('exact process candidate experiment executor', () => { preparationId: input.preparationId, grantDigest: input.grantDigest, closed: true as const, + usageWithinLimits: true as const, calls: [], })), } diff --git a/tests/helpers/candidate-execution-fixture.ts b/tests/helpers/candidate-execution-fixture.ts index b8ba1a1d..c327ff6a 100644 --- a/tests/helpers/candidate-execution-fixture.ts +++ b/tests/helpers/candidate-execution-fixture.ts @@ -379,6 +379,7 @@ export function createCandidateExecutionFixture(active = false): CandidateExecut preparationId, grantDigest: candidateSha('c'), closed: true, + usageWithinLimits: true, calls: [], }), },