From b18292ab1370f4c01d0dacd0d65c25495c89adc0 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 01:29:36 -0600 Subject: [PATCH 1/2] feat(envelope): carry execution-environment identity on sessions and exported spans SessionEnvironment (image, imageDigest, sandboxId, cwd, gitCommit) rides SessionRef, is stamped onto every span, and survives into the OTLP resource block. sandbox-events imports recover it from the wrapper context or the stream's runtime.ready event. Absent fields stay absent: 'environment not captured' remains distinguishable from any captured value, which is what downstream replay eligibility keys on. --- src/attributes.ts | 78 +++++++++++++++++++++++++++++++++++++++ src/file-export.ts | 37 ++++++++++++++++++- src/otlp.ts | 7 ++++ src/session-source.ts | 6 ++- src/types.ts | 22 +++++++++++ tests/file-export.test.ts | 65 ++++++++++++++++++++++++++++++++ tests/shared.test.ts | 25 +++++++++++++ 7 files changed, 238 insertions(+), 2 deletions(-) diff --git a/src/attributes.ts b/src/attributes.ts index 3eae749..e985f79 100644 --- a/src/attributes.ts +++ b/src/attributes.ts @@ -53,11 +53,89 @@ export const ATTR = { CORRUPTION_BYTE_LENGTH: 'traces.session.corruption.byte_length', CORRUPTION_SHA256: 'traces.session.corruption.sha256', RAW_SOURCE_RETENTION: 'traces.session.raw_source_retention', + /** Container image reference the session executed in (OTel semconv key). */ + CONTAINER_IMAGE: 'container.image.name', + /** Immutable image digest (`sha256:…`) — the replay-grade environment pin. */ + CONTAINER_IMAGE_DIGEST: 'container.image.digest', + /** Sandbox/container instance id that executed the session. */ + SANDBOX_ID: 'tangle.sandbox.id', + /** Recorded execution cwd, verbatim. Unlike `tangle.cwd` it is never + * repaired against the host filesystem — replay verification needs the + * in-container path even when no such path exists on this host. */ + ENVIRONMENT_CWD: 'tangle.environment.cwd', } as const /** `tangle.ingest_source` value for CLI-uploaded traces. Wire contract. */ export const INGEST_SOURCE_CLI = 'cli' +/** + * Project a `SessionEnvironment` onto span-attribute keys. Only recorded + * fields are emitted — an absent image stays absent, so "environment not + * captured" and "environment captured without an image" remain different + * artifacts downstream. + */ +export function environmentAttributes( + environment: import('./types.js').SessionEnvironment | undefined, +): Record { + if (!environment) return {} + const attrs: Record = {} + if (environment.image) attrs[ATTR.CONTAINER_IMAGE] = environment.image + if (environment.imageDigest) attrs[ATTR.CONTAINER_IMAGE_DIGEST] = environment.imageDigest + if (environment.sandboxId) attrs[ATTR.SANDBOX_ID] = environment.sandboxId + if (environment.cwd) attrs[ATTR.ENVIRONMENT_CWD] = environment.cwd + if (environment.gitCommit) attrs[ATTR.GIT_COMMIT] = environment.gitCommit + return attrs +} + +/** + * Stamp environment identity onto every span so it survives into the OTLP + * resource block (see `toOpenInferenceSpan`). Additive — a value an adapter + * already set deliberately is never clobbered. + */ +export function stampEnvironmentAttrs( + spans: readonly { attributes: Record }[], + environment: import('./types.js').SessionEnvironment | undefined, +): void { + const attrs = environmentAttributes(environment) + const keys = Object.keys(attrs) + if (keys.length === 0) return + for (const span of spans) { + for (const key of keys) { + if (span.attributes[key] === undefined) span.attributes[key] = attrs[key] + } + } +} + +/** + * Recover a `SessionEnvironment` from stamped span attributes — the read + * counterpart of `stampEnvironmentAttrs`, for consumers that hold only the + * OTLP artifact (replay-environment resolution reads the exported spans, not + * the source session). Returns null when no environment identity was + * recorded — "not captured" stays distinguishable from any captured value. + */ +export function environmentFromSpanAttributes( + spans: readonly { attributes: Readonly> }[], +): import('./types.js').SessionEnvironment | null { + const first = (key: string): string | undefined => { + for (const span of spans) { + const value = span.attributes[key] + if (typeof value === 'string' && value.length > 0) return value + } + return undefined + } + const image = first(ATTR.CONTAINER_IMAGE) + const imageDigest = first(ATTR.CONTAINER_IMAGE_DIGEST) + const sandboxId = first(ATTR.SANDBOX_ID) + const cwd = first(ATTR.ENVIRONMENT_CWD) + if (!image && !imageDigest && !sandboxId && !cwd) return null + return { + ...(image ? { image } : {}), + ...(imageDigest ? { imageDigest } : {}), + ...(sandboxId ? { sandboxId } : {}), + cwd: cwd ?? null, + } +} + /** Harness used when none is specified on a single-harness command. */ export const DEFAULT_HARNESS = 'claude-code' diff --git a/src/file-export.ts b/src/file-export.ts index 7083cf9..ef513d1 100644 --- a/src/file-export.ts +++ b/src/file-export.ts @@ -17,7 +17,8 @@ import { LLM_REASONING_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, } from '@tangle-network/agent-eval/trace-attributes' -import { ATTR, sessionIdFromAttributes } from './attributes.js' +import { ATTR, sessionIdFromAttributes, stampEnvironmentAttrs } from './attributes.js' +import type { SessionEnvironment } from './types.js' import { capText } from './adapters/conversation.js' import { toolIoAttributes } from './adapters/tool-io.js' import { appendAll } from './arrays.js' @@ -403,6 +404,35 @@ function eventKind(row: JsonObject, type: string): { kind: OtlpSpanKind; tool?: return { kind: 'CHAIN' } } +/** + * Environment identity for a recorded sandbox session. Two sources, wrapper + * context winning: explicit `image`/`imageDigest`/`sandboxId`/`cwd` keys on + * the wrapper object, else the stream's own `runtime.ready` event — the + * session-gateway event that names the sandbox and the image it booted. + * Null when the recording carries neither: "environment not captured" must + * stay distinguishable from a captured environment downstream. + */ +function sandboxEventsEnvironment( + context: JsonObject, + rows: readonly JsonObject[], +): SessionEnvironment | null { + const ready = rows.find((row) => eventType(row) === 'runtime.ready') + const readyData = ready && isObject(ready.data) ? ready.data : undefined + const pick = (keys: readonly string[]): string | undefined => + findStringKey(context, keys, 2) ?? (readyData ? findStringKey(readyData, keys, 2) : undefined) + const image = pick(['image']) + const imageDigest = pick(['imageDigest', 'image_digest']) + const sandboxId = pick(['sandboxId', 'sandbox_id']) + const cwd = pick(['cwd', 'workspaceRoot', 'workspace_root']) + if (!image && !imageDigest && !sandboxId && !cwd) return null + return { + ...(image ? { image } : {}), + ...(imageDigest ? { imageDigest } : {}), + ...(sandboxId ? { sandboxId } : {}), + cwd: cwd ?? null, + } +} + function sandboxEventsToSpans(rows: readonly JsonObject[], wrapper?: JsonObject): OtlpSpan[] { const context = wrapper ?? {} const sessionId = @@ -490,6 +520,11 @@ function sandboxEventsToSpans(rows: readonly JsonObject[], wrapper?: JsonObject) })) }) + // Environment identity rides every span so `toOpenInferenceSpan` lifts it + // into the OTLP resource block — the sandbox class is replay-eligible only + // if the image recorded at runtime survives export. + stampEnvironmentAttrs(spans, sandboxEventsEnvironment(context, rows) ?? undefined) + if (!sessionId) return spans return spans.map((item) => ({ ...item, diff --git a/src/otlp.ts b/src/otlp.ts index 93c3a80..2395975 100644 --- a/src/otlp.ts +++ b/src/otlp.ts @@ -266,6 +266,13 @@ export function toOpenInferenceSpan(s: OtlpSpan): Record { for (const k of ['tangle.subject.key', 'git.repository', 'git.branch', 'git.commit', 'tangle.cwd', 'traces.repo_resolution_source']) { if (a[k] != null) resourceAttrs[k] = a[k] } + // Execution-environment identity (see `SessionEnvironment` in types.ts). + // Carried at the resource level because replay verification consumes it per + // session, not per span — without it an eligible sandbox session loses its + // replay eligibility at export time. + for (const k of ['container.image.name', 'container.image.digest', 'tangle.sandbox.id', 'tangle.environment.cwd']) { + if (a[k] != null) resourceAttrs[k] = a[k] + } // How much of the ORIGINAL source is missing from this file. Carried at the // resource level so it survives every further round trip: `readOtlpInput` // merges resource attributes back onto the span, and adds this hop's own diff --git a/src/session-source.ts b/src/session-source.ts index 0b3d32c..9b5d8bf 100644 --- a/src/session-source.ts +++ b/src/session-source.ts @@ -7,7 +7,7 @@ */ import type { OtlpSpan } from './otlp.js' -import { stampSessionIdentity } from './attributes.js' +import { stampEnvironmentAttrs, stampSessionIdentity } from './attributes.js' import { stampSessionIntegrity } from './integrity.js' import { type AdapterSelection, selectAdapters } from './registry.js' import { cwdMatchesSelection, equivalentGitCwds, resolveSessionRepoAttrs, stampRepoAttrs, stampSpanWorkdirRepoAttrs } from './repo.js' @@ -58,6 +58,10 @@ export async function parseSession( if (spans.length === 0) throw new EmptySessionError(ref.path) stampSessionIntegrity(ref, spans) stampSessionIdentity(spans, ref.sessionId) + // Environment identity stamps before repo resolution: recorded ground truth + // (the image/cwd the session actually ran in) outranks host-side inference. + stampEnvironmentAttrs(spans, ref.environment) + if (ref.cwd === null && ref.environment?.cwd) ref.cwd = ref.environment.cwd const repo = await resolveSessionRepoAttrs(ref.cwd, spans) if (repo.cwd) ref.cwd = repo.cwd stampRepoAttrs(spans, repo.attrs) diff --git a/src/types.ts b/src/types.ts index 2fed430..2377bc7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -44,6 +44,26 @@ export interface SessionIntegrity { corruptions: SessionCorruptionReceipt[] } +/** + * Execution-environment identity for a session — what replay verification + * needs to reconstruct where the session's commands actually ran. Populated + * by adapters/importers where known: sandbox sessions know their image + * (`runtime.ready` carries it); host-harness sessions usually do not. + * Absent fields mean "not recorded", never "none". + */ +export interface SessionEnvironment { + /** Container image reference (repo:tag or repo@digest) the session executed in. */ + image?: string + /** Immutable image digest (`sha256:…`) — the replay-grade pin when known. */ + imageDigest?: string + /** Sandbox/container instance id that executed the session. */ + sandboxId?: string + /** Working directory commands executed from; null when unrecorded. */ + cwd: string | null + /** Commit hash the workspace was at when the session ran, when recorded. */ + gitCommit?: string +} + /** A single discovered session, before parsing. */ export interface SessionRef { /** Harness id (matches the nix profile / backend name). */ @@ -58,6 +78,8 @@ export interface SessionRef { mtimeMs: number /** Present when parsing recovered valid records around corrupt source records. */ integrity?: SessionIntegrity + /** Execution-environment identity, when the source records it (sandbox sessions). */ + environment?: SessionEnvironment } export interface LocateOptions { diff --git a/tests/file-export.test.ts b/tests/file-export.test.ts index 900c75e..7a034d0 100644 --- a/tests/file-export.test.ts +++ b/tests/file-export.test.ts @@ -206,6 +206,71 @@ describe('trace evidence export', () => { expect(traceOnly.spans.every((item) => item.attributes['tangle.sessionId'] === undefined)).toBe(true) }) + it('stamps wrapper environment identity onto every span and lifts it into the OTLP resource', () => { + const result = exportTraceEvidenceText(JSON.stringify({ + session_id: 'session-env-1', + image: 'ghcr.io/tangle-network/sandbox:base', + imageDigest: 'sha256:' + 'c'.repeat(64), + sandboxId: 'sbx-env-1', + cwd: '/workspace/repo', + events: [ + { type: 'start', timestamp: '2026-08-04T00:00:00.000Z' }, + { type: 'tool-invocation', toolName: 'bash', input: 'ls', timestamp: '2026-08-04T00:00:01.000Z' }, + ], + }), { format: 'sandbox-events' }) + for (const item of result.spans) { + expect(item.attributes).toEqual(expect.objectContaining({ + 'container.image.name': 'ghcr.io/tangle-network/sandbox:base', + 'container.image.digest': 'sha256:' + 'c'.repeat(64), + 'tangle.sandbox.id': 'sbx-env-1', + 'tangle.environment.cwd': '/workspace/repo', + })) + } + const rows = parseRows(serializeSpans(result.spans)) + for (const row of rows) { + const resource = row.resource as { attributes: Record } + expect(resource.attributes).toEqual(expect.objectContaining({ + 'container.image.name': 'ghcr.io/tangle-network/sandbox:base', + 'container.image.digest': 'sha256:' + 'c'.repeat(64), + 'tangle.sandbox.id': 'sbx-env-1', + 'tangle.environment.cwd': '/workspace/repo', + })) + } + }) + + it('falls back to the runtime.ready event for environment identity and omits it when absent', () => { + const fromReady = exportTraceEvidenceRows([ + { + type: 'runtime.ready', + data: { + timestamp: '2026-08-04T00:00:00.000Z', + image: 'ghcr.io/tangle-network/sandbox:ready', + sandboxId: 'sbx-ready-1', + workspaceRoot: '/workspace/from-ready', + }, + }, + { type: 'done', data: { timestamp: '2026-08-04T00:00:01.000Z' } }, + ], { format: 'sandbox-events' }) + for (const item of fromReady.spans) { + expect(item.attributes).toEqual(expect.objectContaining({ + 'container.image.name': 'ghcr.io/tangle-network/sandbox:ready', + 'tangle.sandbox.id': 'sbx-ready-1', + 'tangle.environment.cwd': '/workspace/from-ready', + })) + expect(item.attributes['container.image.digest']).toBeUndefined() + } + + const uncaptured = exportTraceEvidenceRows([ + { type: 'start', data: { timestamp: '2026-08-04T00:00:00.000Z' } }, + ], { format: 'sandbox-events' }) + for (const item of uncaptured.spans) { + expect(item.attributes['container.image.name']).toBeUndefined() + expect(item.attributes['container.image.digest']).toBeUndefined() + expect(item.attributes['tangle.sandbox.id']).toBeUndefined() + expect(item.attributes['tangle.environment.cwd']).toBeUndefined() + } + }) + it('writes an exported file from JSONL input', async () => { const dir = await mkdtemp(join(tmpdir(), 'traces-export-test-')) const input = join(dir, 'policy.jsonl') diff --git a/tests/shared.test.ts b/tests/shared.test.ts index 4902324..fe8688a 100644 --- a/tests/shared.test.ts +++ b/tests/shared.test.ts @@ -166,3 +166,28 @@ describe('parseIsoToEpochMs', () => { expect(() => parseIsoToEpochMs('999999999999999999999999')).toThrow(/invalid timestamp/) }) }) + +describe('environment identity attributes', () => { + it('round-trips a SessionEnvironment through span attributes without clobbering adapter values', async () => { + const { environmentFromSpanAttributes, stampEnvironmentAttrs } = await import('../src/attributes.js') + const environment = { + image: 'ghcr.io/tangle-network/sandbox:base', + imageDigest: `sha256:${'d'.repeat(64)}`, + sandboxId: 'sbx-rt-1', + cwd: '/workspace/repo', + } + const spans = [ + { attributes: { 'container.image.name': 'adapter-set:keep' } as Record }, + { attributes: {} as Record }, + ] + stampEnvironmentAttrs(spans, environment) + expect(spans[0]!.attributes['container.image.name']).toBe('adapter-set:keep') + expect(spans[1]!.attributes['container.image.name']).toBe('ghcr.io/tangle-network/sandbox:base') + expect(environmentFromSpanAttributes([spans[1]!])).toEqual(environment) + }) + + it('returns null when no environment identity was ever recorded', async () => { + const { environmentFromSpanAttributes } = await import('../src/attributes.js') + expect(environmentFromSpanAttributes([{ attributes: { unrelated: 'x' } }])).toBeNull() + }) +}) From 7859d0b16ac0c80c714fccdad1cf4494526edc32 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 4 Aug 2026 02:08:17 -0600 Subject: [PATCH 2/2] feat(envelope): carry sandbox LLM-event model and token usage onto exported spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sandbox-events recordings put usage on the llm event row (OpenAI usage.prompt_tokens, opencode tokenUsage.inputTokens, bare snake_case); the converter dropped all of them, so an imported sandbox session lost token accounting and cost attribution at export time. LLM-kind rows now map model + input/output tokens + cost onto the span's canonical attributes. Absent counts stay absent — no invented zeros. --- src/file-export.ts | 45 +++++++++++++++++++++++++++++++++++++++ tests/file-export.test.ts | 23 ++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/file-export.ts b/src/file-export.ts index ef513d1..9e41e50 100644 --- a/src/file-export.ts +++ b/src/file-export.ts @@ -433,6 +433,50 @@ function sandboxEventsEnvironment( } } +/** + * Model identity and token usage recorded on one LLM event row, across the + * key spellings sandbox recorders use (`usage.prompt_tokens` OpenAI-style, + * `tokenUsage.inputTokens` opencode-style, bare snake_case). Absent values + * stay absent — a zero here would claim a measured count that never was. + */ +function llmEventUsage(row: JsonObject): { + model?: string + inputTokens?: number + outputTokens?: number + costUsd?: number +} { + const num = (keys: readonly string[]): number | undefined => { + for (const key of keys) { + const direct = numberValue(row[key]) + if (direct !== undefined) return direct + for (const nest of ['data', 'usage', 'tokenUsage']) { + const container = row[nest] + if (!isObject(container)) continue + const value = numberValue(container[key]) + if (value !== undefined) return value + for (const inner of ['usage', 'tokenUsage']) { + const deep = container[inner] + if (isObject(deep)) { + const deepValue = numberValue(deep[key]) + if (deepValue !== undefined) return deepValue + } + } + } + } + return undefined + } + const model = findStringKey(row, ['model'], 3) + const inputTokens = num(['prompt_tokens', 'input_tokens', 'inputTokens']) + const outputTokens = num(['completion_tokens', 'output_tokens', 'outputTokens']) + const costUsd = num(['totalCostUsd', 'total_cost_usd', 'costUsd', 'cost_usd']) + return { + ...(model ? { model } : {}), + ...(inputTokens !== undefined ? { inputTokens } : {}), + ...(outputTokens !== undefined ? { outputTokens } : {}), + ...(costUsd !== undefined ? { costUsd } : {}), + } +} + function sandboxEventsToSpans(rows: readonly JsonObject[], wrapper?: JsonObject): OtlpSpan[] { const context = wrapper ?? {} const sessionId = @@ -517,6 +561,7 @@ function sandboxEventsToSpans(rows: readonly JsonObject[], wrapper?: JsonObject) step: index, content: capText(stableJson(row)), extra, + ...(kind === 'LLM' ? llmEventUsage(row) : {}), })) }) diff --git a/tests/file-export.test.ts b/tests/file-export.test.ts index 7a034d0..24f1601 100644 --- a/tests/file-export.test.ts +++ b/tests/file-export.test.ts @@ -238,6 +238,29 @@ describe('trace evidence export', () => { } }) + it('carries LLM event model and token usage onto the span, and never invents zeros', () => { + const result = exportTraceEvidenceRows([ + { + type: 'llm.completion', + data: { + timestamp: '2026-08-04T00:00:00.000Z', + model: 'glm-5.2', + usage: { prompt_tokens: 1200, completion_tokens: 340 }, + }, + }, + { type: 'llm.completion', data: { timestamp: '2026-08-04T00:00:01.000Z' } }, + ], { format: 'sandbox-events' }) + const llmSpans = result.spans.filter((item) => item.attributes['openinference.span.kind'] === 'LLM') + expect(llmSpans).toHaveLength(2) + expect(llmSpans[0]!.attributes).toEqual(expect.objectContaining({ + 'llm.model_name': 'glm-5.2', + 'llm.token_count.prompt': 1200, + 'llm.token_count.completion': 340, + })) + expect(llmSpans[1]!.attributes['llm.token_count.prompt']).toBeUndefined() + expect(llmSpans[1]!.attributes['llm.token_count.completion']).toBeUndefined() + }) + it('falls back to the runtime.ready event for environment identity and omits it when absent', () => { const fromReady = exportTraceEvidenceRows([ {