From f8405c5b2fb5e74d7ab876651c9908d4588080d8 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 12 Aug 2026 11:28:31 -0600 Subject: [PATCH] feat(sandbox): add fail-closed retained cloud sessions --- .../2026-08-12-tangle-retained-canary.md | 53 ++ .../2026-08-12T17-22-32Z/findings.jsonl | 7 + .../2026-08-12T17-22-32Z/manifest.json | 26 + .../2026-08-12T17-22-32Z/summary.md | 41 ++ .agent/skill-runs.jsonl | 10 + README.md | 28 +- ...ry-post-hardening-production-20260812.json | 85 +++ ...d-retained-canary-production-20260812.json | 493 +++++++++++++ ...ed-post-hardening-production-20260812.json | 214 ++++++ ...ed-safe-preflight-production-20260812.json | 200 +++++ docs/01-product-contract.md | 17 +- docs/03-architecture.md | 26 + docs/04-runtime-contracts.md | 75 +- docs/05-profiles-and-connections.md | 30 +- docs/06-conversations-forks-and-analysis.md | 4 +- docs/08-verification.md | 92 ++- docs/09-delivery-plan.md | 1 + docs/10-upstream-strategy.md | 4 +- docs/decisions/002-runtime-boundary.md | 2 + docs/decisions/003-local-event-journal.md | 4 + .../004-application-effect-coordination.md | 4 + package.json | 6 +- pnpm-lock.yaml | 84 +-- pnpm-workspace.yaml | 12 +- scripts/live-bridge/capture.mjs | 2 +- scripts/live-bridge/redaction.mjs | 2 +- scripts/live-demo.test.mjs | 1 + scripts/live-demo/public-safety.mjs | 2 +- scripts/live-required.test.mjs | 132 +++- scripts/live-required/headless.mjs | 249 +++++-- .../tangle-sandbox-braid-soak.mjs | 681 +++++++++++++++++- .../tangle-sandbox-braid-stress-support.mjs | 137 +++- .../tangle-sandbox-braid-stress.mjs | 350 ++++++++- scripts/release/redaction.mjs | 6 +- scripts/release/release-collector.test.mjs | 1 + .../production-connection-providers.ts | 42 +- .../production-connection-types.ts | 17 +- .../connections/production-connections.ts | 2 + src/adapters/runtime/prepared-execution.ts | 1 + .../runtime/production-backend-resolver.ts | 23 +- .../production-tangle-sandbox-backend.ts | 212 +++++- .../runtime/retained-execution-contract.ts | 2 + .../runtime/retained-execution-projection.ts | 21 +- src/adapters/runtime/retained-execution.ts | 14 +- .../runtime/tangle-retained-execution.ts | 33 + src/adapters/runtime/tangle-retained-run.ts | 111 +++ .../runtime/tangle-sandbox-retention.ts | 93 +++ src/app/connections.ts | 1 + src/app/production-composition.ts | 39 +- src/app/provider-values.ts | 23 + src/app/run-execution.ts | 6 +- src/app/run-replay.ts | 20 +- src/domain/entities-core.ts | 8 + src/domain/invariants-profile.ts | 44 +- src/domain/secret-sanitizer.ts | 4 +- test/application.test.ts | 14 + test/domain-invariants.test.ts | 8 + test/domain-text.test.ts | 13 + test/production-connection-actions.test.ts | 2 + test/production-connection-setup.test.ts | 2 + test/retained-execution-lifecycle.test.ts | 44 ++ test/support/tangle-retained-sandbox.ts | 381 ++++++++++ test/tangle-retained-lifecycle.test.ts | 323 +++++++++ test/tangle-retained-restart.test.ts | 205 ++++++ .../tangle-sandbox-braid-soak-script.test.mjs | 485 +++++++++++-- ...ngle-sandbox-braid-stress-runtime.test.mjs | 186 +++++ ...angle-sandbox-braid-stress-script.test.mjs | 145 ++++ test/w8-runs.test.ts | 68 ++ 68 files changed, 5315 insertions(+), 358 deletions(-) create mode 100644 .agent/autopsies/2026-08-12-tangle-retained-canary.md create mode 100644 .agent/critical-audit/2026-08-12T17-22-32Z/findings.jsonl create mode 100644 .agent/critical-audit/2026-08-12T17-22-32Z/manifest.json create mode 100644 .agent/critical-audit/2026-08-12T17-22-32Z/summary.md create mode 100644 artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json create mode 100644 artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json create mode 100644 artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json create mode 100644 artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json create mode 100644 src/adapters/runtime/tangle-retained-execution.ts create mode 100644 src/adapters/runtime/tangle-retained-run.ts create mode 100644 src/adapters/runtime/tangle-sandbox-retention.ts create mode 100644 test/support/tangle-retained-sandbox.ts create mode 100644 test/tangle-retained-lifecycle.test.ts create mode 100644 test/tangle-retained-restart.test.ts create mode 100644 test/tangle-sandbox-braid-stress-runtime.test.mjs diff --git a/.agent/autopsies/2026-08-12-tangle-retained-canary.md b/.agent/autopsies/2026-08-12-tangle-retained-canary.md new file mode 100644 index 0000000..58acb3b --- /dev/null +++ b/.agent/autopsies/2026-08-12-tangle-retained-canary.md @@ -0,0 +1,53 @@ +# Tangle cloud canary autopsy + +## Run + +The production checks used Braid commit `8f066cc` plus the retained-session worktree. +They used OpenCode, `tangle-router/glm-5.2`, Runtime through `0.132.12`, and Sandbox SDK `0.21.1`. +The checks used fresh bounded credentials that were revoked after each check. + +## Verified findings + +- The post-hardening retained canary failed before allocation in 4.695 seconds. +- The published provider lacks exact lookup after an unacknowledged dispatch. +- Active Sandboxes stayed at four. +- Total Sandboxes stayed at 9,241. +- Compute, GPU, and cost counters had zero measured change. +- The exact Braid owner-tag query matched zero resources. +- An earlier retained diagnostic reached an older sidecar and was rejected for its `runControlRef` field. +- That earlier attempt moved total Sandboxes from 9,239 to 9,240 but returned active count to four. +- A later ephemeral Braid canary timed out after 182.581 seconds with no environment identifier. +- The Braid run remained `streaming` after Runtime emitted only `backend_start`. +- A direct SDK create with the same route failed in 2.177 seconds. +- The outer response was HTTP 400 with `CONFIG_ERROR`. +- The nested response was HTTP 403 with `Service "sandbox" is not authorized for this endpoint`. +- The same key authenticated Sandbox usage and Router model discovery. +- Direct SDK account counts stayed at four active and 9,241 total Sandboxes. +- Runtime `0.132.12` still treats the permanent error as retryable because its message contains `provision failed`. +- Sandbox npm `latest` points at `0.21.0`, but Runtime `0.132.12` requires Sandbox `>=0.21.1 <0.22.0`. + +## Classification + +Retained mode is safely unavailable because the provider does not supply exact recovery lookup. +Ephemeral cloud creation is currently blocked by internal platform authorization. +Runtime then hides that terminal failure behind an excessive retry. +The npm tag can also install an incompatible SDK version. +These are three independent upstream failures. +Braid created no resource during either post-hardening canary. + +## Action + +Issue `tangle-network/agent-dev-container#5249` tracks the earlier missing-turn response mismatch. +Issue `tangle-network/agent-dev-container#5251` tracks the retained sidecar rollout. +Issue `tangle-network/agent-dev-container#5277` tracks the current platform authorization failure. +Issue `tangle-network/agent-dev-container#5278` tracks the incorrect npm tag. +Issue `tangle-network/agent-runtime#808` tracks permanent-error retry classification. +Braid keeps retained mode fail-closed and defaults new Sandbox connections to ephemeral mode. +The production canary must pass before the bounded durability cohort starts. + +## Artifacts + +- `artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json` +- `artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json` +- `artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json` +- `artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json` diff --git a/.agent/critical-audit/2026-08-12T17-22-32Z/findings.jsonl b/.agent/critical-audit/2026-08-12T17-22-32Z/findings.jsonl new file mode 100644 index 0000000..8de785d --- /dev/null +++ b/.agent/critical-audit/2026-08-12T17-22-32Z/findings.jsonl @@ -0,0 +1,7 @@ +{"id":1,"severity":"HIGH","file":"scripts/live-required/tangle-sandbox-braid-stress.mjs","line":594,"defect":"Public cloud proof serialized raw customer and billing-owner identifiers.","scenario":"An authenticated live proof writes its account observation, then a committed public artifact discloses two real account identifiers.","status":"measured · resolved","evidence":"scripts/live-required/tangle-sandbox-braid-stress.mjs:588-594; actual artifact scan → 5/5 JSON files contain zero customerId or billingOwnerId keys","fix":"Compare raw identity only in memory and emit one SHA-256 identity digest.","verification":"public artifact credential-pattern scan → 5/5 pass; raw-account-key scan → 5/5 pass; pnpm check → 681 pass, 2 skip, 0 fail","costIfShipped":"2 account identifiers disclosed per published proof artifact","savedIfFixed":"2 of 2 account identifiers removed per artifact"} +{"id":2,"severity":"HIGH","file":"src/adapters/runtime/retained-execution.ts","line":75,"defect":"Detach during an unresolved cloud start reopened the event reader after start completed.","scenario":"A user detaches while start is pending, the provider returns a handle, then Braid reads a run the user already detached and can duplicate reconnect delivery.","status":"measured · resolved","evidence":"src/adapters/runtime/retained-execution.ts:66-76; test/retained-execution-lifecycle.test.ts:206-245","fix":"Persist detached state before aborting and stop streamTurn after the start promise resolves.","verification":"pnpm check → in-flight detach test passed; 681 pass, 2 skip, 0 fail","costIfShipped":"1 unintended reader per in-flight detached run","savedIfFixed":"1 of 1 unintended reader prevented per run"} +{"id":3,"severity":"MEDIUM","file":"src/adapters/runtime/tangle-sandbox-retention.ts","line":53,"defect":"The retained client wrapper dropped identity, usage, and subscription methods.","scenario":"A retained connection wraps a Sandbox client, then Braid cannot observe account identity or spend and reports three available provider surfaces as unknown.","status":"measured · resolved","evidence":"src/adapters/runtime/tangle-sandbox-retention.ts:53-55,89-91; test/tangle-retained-lifecycle.test.ts:142-173","fix":"Bind and preserve every optional account observation method on the wrapper.","verification":"pnpm check → retained account-observation test passed; 681 pass, 2 skip, 0 fail","costIfShipped":"3 account observation methods lost per wrapped client","savedIfFixed":"3 of 3 methods preserved per client"} +{"id":4,"severity":"MEDIUM","file":"scripts/live-required/tangle-sandbox-braid-stress.mjs","line":926,"defect":"A lost shutdown acknowledgement skipped process-tree cleanup proof.","scenario":"Shutdown RPC fails after delivery, then the stress runner throws before checking child exit and cannot prove that descendants stopped.","status":"measured · resolved","evidence":"scripts/live-required/tangle-sandbox-braid-stress.mjs:926-947; test/tangle-sandbox-braid-stress-runtime.test.mjs:123-157","fix":"Always close and verify the process tree, then attach that proof to the original shutdown error.","verification":"pnpm check → shutdown-failure cleanup test passed; live-required self-test 26/26 passed","costIfShipped":"1 process tree unverified per shutdown acknowledgement failure","savedIfFixed":"1 of 1 process tree verified per failure"} +{"id":5,"severity":"MEDIUM","file":"src/app/run-replay.ts","line":53,"defect":"Recovery replaced a typed dispatch failure when exact provider status was unavailable.","scenario":"Cloud provisioning returns CLOUD_PROVISION_REJECTED and status lookup is absent, then Braid shows a generic unknown-state message instead of the safe typed cause.","status":"measured · resolved","evidence":"src/app/run-replay.ts:53-55,76-82,104-110; test/w8-runs.test.ts:251-270","fix":"Carry the sanitized prior failure through reconnect and null-status reconciliation.","verification":"pnpm check → typed diagnostic test passed; 681 pass, 2 skip, 0 fail","costIfShipped":"1 actionable diagnostic lost per affected failed dispatch","savedIfFixed":"1 of 1 diagnostic retained per affected dispatch"} +{"id":6,"severity":"MEDIUM","file":"src/domain/invariants-profile.ts","line":140,"defect":"Non-Sandbox connections accepted Sandbox lifecycle fields.","scenario":"A CLI Bridge connection includes retained lifecycle and idle TTL, then validation accepts provider options that its adapter cannot implement.","status":"measured · resolved","evidence":"src/domain/invariants-profile.ts:140-145; test/domain-invariants.test.ts:199-206","fix":"Reject lifecycle and idle TTL unless the connection kind is tangle-sandbox.","verification":"pnpm check → domain invariant test passed; 681 pass, 2 skip, 0 fail","costIfShipped":"2 unsupported fields accepted per malformed connection","savedIfFixed":"2 of 2 unsupported fields rejected per connection"} +{"id":7,"severity":"LOW","file":"docs/05-profiles-and-connections.md","line":275,"defect":"Documentation claimed an operator-selectable retained control that the connection editor does not expose.","scenario":"A terminal user follows the setup documentation, then no retained lifecycle control exists in the five-field editor.","status":"measured · resolved","evidence":"src/views/tui/connection-metadata-editor-model.ts:15-21; docs/05-profiles-and-connections.md:273-277","fix":"Describe retained lifecycle as a connection configuration option until the provider path and editor control are available.","verification":"rg connection editor fields → kind, name, endpoint, region, account only; pnpm format:check passed","costIfShipped":"1 unavailable UI action claimed per reader path","savedIfFixed":"1 of 1 unavailable action claims removed"} diff --git a/.agent/critical-audit/2026-08-12T17-22-32Z/manifest.json b/.agent/critical-audit/2026-08-12T17-22-32Z/manifest.json new file mode 100644 index 0000000..cce201b --- /dev/null +++ b/.agent/critical-audit/2026-08-12T17-22-32Z/manifest.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 1, + "scope": "Braid retained Tangle Sandbox lifecycle, stress proof, and public evidence", + "files": 65, + "base": "8f066cc20c750579015bf47589d3d742d6bb0bc5", + "head": "8f066cc20c750579015bf47589d3d742d6bb0bc5", + "workingTree": true, + "diffSha256": "2389bc43d72a13cf9bbd348d57d8f7a9955b2673d844a3953d5856f3c4c9b067", + "projectType": "Node.js ESM package", + "reviewers": [ + "A: retained lifecycle correctness, identity, cleanup, credentials, and races", + "B: runtime boundaries, module ownership, replay, and failure handling", + "C: callsites, documentation accuracy, dependency hygiene, and real-system proof" + ], + "mode": "current working-tree review with serial local checks", + "verdict": "APPROVE", + "counts": { + "critical": 0, + "high": 2, + "medium": 4, + "low": 1, + "resolved": 7, + "unresolved": 0, + "dropped": 0 + } +} diff --git a/.agent/critical-audit/2026-08-12T17-22-32Z/summary.md b/.agent/critical-audit/2026-08-12T17-22-32Z/summary.md new file mode 100644 index 0000000..5d54445 --- /dev/null +++ b/.agent/critical-audit/2026-08-12T17-22-32Z/summary.md @@ -0,0 +1,41 @@ +# Audit: Braid retained Tangle Sandbox lifecycle and proof — 8f066cc20c750579015bf47589d3d742d6bb0bc5..8f066cc20c750579015bf47589d3d742d6bb0bc5 — n=65 files, 7 findings + +**Verdict:** APPROVE — 7 of 7 measured defects resolved · 0 CRITICAL / 2 HIGH / 4 MEDIUM / 1 LOW +**Worst:** #1 `scripts/live-required/tangle-sandbox-braid-stress.mjs:594` — public proof disclosed two account identifiers · cost if shipped 2 identifiers per artifact +**Next:** `/verify` + +## Scope + +| Field | Value | +|---|---| +| Files | n=65 via current working-tree diff | +| Base..head | `8f066cc20c750579015bf47589d3d742d6bb0bc5..8f066cc20c750579015bf47589d3d742d6bb0bc5` | +| Project type | Node.js ESM package | +| Reviewers | A,B,C · serial | +| Not inspected | Provider implementation outside Braid; live retained execution remains blocked by tracked upstream issues. | + +## Findings — 7 of 7, ranked + +| # | Sev | file:line | Defect | Failure scenario (input/state → wrong result) | Status | Evidence | Fix | Verification | Cost if shipped | Saved if fixed | +|---:|---|---|---|---|---|---|---|---|---:|---:| +| 1 | HIGH | `scripts/live-required/tangle-sandbox-braid-stress.mjs:594` | Public proof emitted raw account identity. | Authenticated proof → two identifiers published. | measured · resolved | actual artifacts: 5/5 have zero raw identity keys | Emit one digest. | public scans 5/5 pass | 2 identifiers/artifact | 2/2 removed | +| 2 | HIGH | `src/adapters/runtime/retained-execution.ts:75` | Pending detach reopened the event reader. | Detach before start resolves → detached run streams locally. | measured · resolved | `test/retained-execution-lifecycle.test.ts:206` | Stop after start when detached. | `pnpm check` test passed | 1 reader/run | 1/1 prevented | +| 3 | MEDIUM | `src/adapters/runtime/tangle-sandbox-retention.ts:53` | Wrapper dropped account methods. | Wrap client → identity, usage, and subscription become unknown. | measured · resolved | `test/tangle-retained-lifecycle.test.ts:142` | Preserve bound methods. | `pnpm check` test passed | 3 methods/client | 3/3 preserved | +| 4 | MEDIUM | `scripts/live-required/tangle-sandbox-braid-stress.mjs:926` | Shutdown error hid cleanup proof. | Lost acknowledgement → descendants remain unverified. | measured · resolved | `test/tangle-sandbox-braid-stress-runtime.test.mjs:123` | Verify close before rethrow. | live-required 26/26 | 1 tree/failure | 1/1 verified | +| 5 | MEDIUM | `src/app/run-replay.ts:53` | Recovery dropped the typed cause. | Failed dispatch plus no status → generic error. | measured · resolved | `test/w8-runs.test.ts:251` | Carry safe prior detail. | `pnpm check` test passed | 1 diagnostic/run | 1/1 retained | +| 6 | MEDIUM | `src/domain/invariants-profile.ts:140` | Other connections accepted Sandbox lifecycle fields. | CLI Bridge plus retained TTL → invalid configuration accepted. | measured · resolved | `test/domain-invariants.test.ts:199` | Restrict fields by kind. | `pnpm check` test passed | 2 fields/config | 2/2 rejected | +| 7 | LOW | `docs/05-profiles-and-connections.md:275` | Docs claimed a missing editor action. | User follows setup text → no retained control exists. | measured · resolved | `src/views/tui/connection-metadata-editor-model.ts:15` | State configuration boundary. | editor field inspection | 1 claim/reader | 1/1 removed | + +0 dropped (no reproducible failure scenario / no actionable fix). + +## Assumptions & unverified + +| Assumption | Finding it would flip | Check that settles it | +|---|---|---| +| The provider will add exact retained-run lookup without changing the six-field control reference. | The current adapter contract could require a new shared field. | Resolve upstream runtime issue 800 and provider issue 146, then rerun the retained stress command. | +| The current Sandbox authorization regression is external to Braid. | A Braid credential-routing defect could be present. | Resolve ADC issue 5277, then rerun both direct SDK and Braid canaries with one new key. | + +## Self-gate + +9/9 passed — failed: none. +1 verdict = decision + 1 number · 2 every finding has file:line · 3 concrete failure scenario · 4 status label · 5 evidence is a pointer · 6 cost both sides · 7 fix + verification per row · 8 zero adjectives standing in for counts · 9 148 words ≤600 outside tables. diff --git a/.agent/skill-runs.jsonl b/.agent/skill-runs.jsonl index 968b6b7..c4d0017 100644 --- a/.agent/skill-runs.jsonl +++ b/.agent/skill-runs.jsonl @@ -41,3 +41,13 @@ {"skill":"/harden","ts":"2026-08-12T03:42:22Z","project":"braid-transcript-first-wt","target":"provider-neutral retained execution lifecycle, exact provider identity, recovery, cancellation, and event binding n=11 files","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/critical-audit","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/critical-audit","ts":"2026-08-12T03:42:23Z","project":"braid-transcript-first-wt","target":"provider-neutral retained execution lifecycle and provider-owned identity binding n=11 files","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/converge","ts":"2026-08-12T12:53:14Z","project":"braid-transcript-first-wt","target":"Braid full release check performance failure","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/review-to-green","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/sandbox-sdk-integration","ts":"2026-08-12T14:30:51Z","project":"braid-retained-v2-wt","target":"Braid retained Tangle sandbox execution through the published SDK and provider","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/ground-truth","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/ground-truth","ts":"2026-08-12T14:30:51Z","project":"braid-retained-v2-wt","target":"production retained cloud create, dispatch, reconnect, cleanup, usage, and latency path n=1","operatorPrompt":"","durationMin":null,"verdict":"EXTERNAL_BLOCKER_CONFIRMED","dispatchedTo":"/diagnose","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/diagnose","ts":"2026-08-12T14:30:51Z","project":"braid-retained-v2-wt","target":"retained Tangle canary failure n=1","operatorPrompt":"","durationMin":null,"verdict":"ROOT_CAUSE_CONFIRMED","dispatchedTo":"/autopsy","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/autopsy","ts":"2026-08-12T14:30:51Z","project":"braid-retained-v2-wt","target":"production retained canary and direct start traces n=2","operatorPrompt":"","durationMin":null,"verdict":"INFRA_BUG","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/bp-ops","ts":"2026-08-12T14:30:51Z","project":"braid-retained-v2-wt","target":"Sandbox production health, deploy history, and resource cleanup","operatorPrompt":"","durationMin":null,"verdict":"DEPLOY_BLOCKED","dispatchedTo":"/adc-infra-triage","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/adc-infra-triage","ts":"2026-08-12T14:30:52Z","project":"braid-retained-v2-wt","target":"production and staging ADC revisions versus exact retained-run revision","operatorPrompt":"","durationMin":null,"verdict":"ROOT_CAUSE_CONFIRMED","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/report","ts":"2026-08-12T14:30:52Z","project":"braid-retained-v2-wt","target":"Braid retained Tangle production status and stress evidence","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/harden","ts":"2026-08-12T17:28:15Z","project":"braid-retained-v2-wt","target":"Braid retained Tangle lifecycle, public proof redaction, replay races, and cleanup n=65 files","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/critical-audit","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/critical-audit","ts":"2026-08-12T17:28:16Z","project":"braid-retained-v2-wt","target":"Braid retained Tangle lifecycle, stress proof, and public evidence n=65 files","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-08-12T17:28:16Z","project":"braid-retained-v2-wt","target":"Braid retained Tangle source, 683 tests, packed terminal, cloud proof contracts, and CLI Bridge matrix","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/README.md b/README.md index afa955a..167234c 100644 --- a/README.md +++ b/README.md @@ -223,22 +223,42 @@ Changing runners creates a new provider session with an explicit portable-contex It does not claim to transfer hidden process memory, runner-specific todos, opaque tool state, or native session internals. -A Tangle sandbox connection can provide an isolated remote workspace, environment lifecycle, checkpoint, fork, replay, and resource metadata when its capabilities report those operations. +A Tangle sandbox connection provides an isolated remote workspace and reports only the lifecycle, replay, control, and resource capabilities that the current provider proves. Braid shows those capabilities and their receipts through the same activity and graph surfaces. -The current published Tangle path runs one isolated cloud turn and deletes its environment after the turn. +New Tangle Sandbox connections default to one ephemeral cloud turn and delete that environment after the turn. -Cloud restart, retained-session continuation, checkpoint, and fork remain unavailable until the shared provider reports exact recovery support. +Retained lifecycle is an explicit connection configuration option with a bounded idle limit. + +Before retained execution creates a sandbox, Braid requires exact control plus provider-backed lookup for an unacknowledged dispatch. + +The current published provider does not report that complete contract or lookup, so Braid rejects retained mode without creating a resource. + +When the provider reports both, a fresh Braid process can recover before or after the six-field reference commits. + +Native follow-up turns remain disabled until the provider also proves that its context boundary matches Braid's recorded boundary. + +Checkpoint, environment fork, and interaction response remain unavailable until the shared provider reports and proves those operations. The user can inspect the requested and verified execution location, but provider-private machine details remain unavailable when they are not reported. -The latest production stress proof completed 20 of 20 Braid turns through OpenCode, GLM 5.2, and Tangle Sandbox at four-way concurrency. +The latest passing production stress proof completed 20 of 20 ephemeral Braid turns through OpenCode, GLM 5.2, and Tangle Sandbox at four-way concurrency. All 20 remote environments were unique and confirmed deleted, while the account's active Sandbox count returned from four to four. See the [secret-free proof artifact](artifacts/verification/live/tangle-sandbox-braid-execution-stress-production-20260812.json) for every run, token receipt, latency, environment observation, and cleanup result. +A later canary found a current platform regression before environment allocation. + +Fresh credentials authenticated Sandbox and Router, but the internal model-key step rejected the Sandbox service with HTTP 403. + +Braid left zero owned environments, while [ADC issue 5277](https://github.com/tangle-network/agent-dev-container/issues/5277) tracks the platform failure. + +[Runtime issue 808](https://github.com/tangle-network/agent-runtime/issues/808) tracks the separate ten-minute retry of that permanent rejection. + +The [verification record](docs/08-verification.md#current-core-path-observations) keeps the full results, limits, and tracked platform work. + ## Commands users reach for first | Need | Command or key | diff --git a/artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json b/artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json new file mode 100644 index 0000000..b5da06e --- /dev/null +++ b/artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": "braid.tangle-sandbox-execution-soak.v1", + "status": "failed", + "proofId": "1786552467896-0a06399a", + "startedAt": "2026-08-12T16:34:27.896Z", + "completedAt": "2026-08-12T16:37:34.793Z", + "requestedRuns": 1, + "attemptedRuns": 1, + "concurrency": 1, + "failures": [ + "run 1 failed", + "run 1 lacked automatic cleanup" + ], + "config": { + "endpoint": { + "scheme": "https", + "host": "sandbox.tangle.tools" + }, + "connectionId": "connection-live-tangle-sandbox", + "connectionKind": "tangle-sandbox", + "credentialConfigured": true, + "model": "tangle-router/glm-5.2", + "runner": "opencode", + "provider": "tangle-router" + }, + "account": { + "stable": true, + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46" + }, + "usage": { + "before": { + "computeMinutes": 1052, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "after": { + "computeMinutes": 1056, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "delta": { + "activeSandboxes": 0, + "totalSandboxes": 0, + "computeMinutes": 4, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "unknownFields": [] + } + }, + "latencyMs": { + "n": 1, + "min": 182581.257125, + "median": 182581.257125, + "p90": 182581.257125, + "max": 182581.257125 + }, + "attempts": [ + { + "index": 0, + "status": "failed", + "elapsedMs": 182581.257125, + "environmentId": null, + "automaticCleanup": false, + "error": { + "name": "LiveBridgeError", + "message": "timed out waiting for turn terminal state" + }, + "cleanup": { + "status": "passed", + "matchedCount": 0, + "remainingCount": 0 + } + } + ] +} diff --git a/artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json b/artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json new file mode 100644 index 0000000..ac896e3 --- /dev/null +++ b/artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json @@ -0,0 +1,493 @@ +{ + "schemaVersion": "braid.tangle-sandbox-braid-soak.v1", + "status": "failed", + "startedAt": "2026-08-12T14:08:10.841Z", + "completedAt": "2026-08-12T14:08:35.679Z", + "requestedRuns": 1, + "attemptedRuns": 1, + "concurrency": 1, + "stoppedAfterCanary": true, + "failures": [ + "run 1 did not pass exact proof", + "cloud environment identity was missing", + "Braid binary digest was missing" + ], + "cleanup": { + "exactProofs": 0, + "exactResourcesRemaining": 1, + "activeResourceDelta": 0, + "activeResourceDeltas": [ + 0 + ] + }, + "accountUsage": { + "complete": true, + "before": { + "computeMinutes": 906, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9239, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "after": { + "computeMinutes": 906, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9240, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "delta": { + "activeSandboxes": 0, + "totalSandboxes": 1, + "computeMinutes": 0, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "unknownFields": [] + } + }, + "latency": { + "totalMs": { + "n": 1, + "min": 24835.949061000003, + "median": 24835.949061000003, + "p90": 24835.949061000003, + "max": 24835.949061000003 + }, + "phases": {} + }, + "sessionSpend": { + "scope": "every unique local run in every cloud proof", + "rows": [], + "tokens": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "input": 0, + "output": 0 + }, + "cost": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "usd": 0 + } + }, + "attempts": [ + { + "index": 0, + "completionSequence": 0, + "startedAt": "2026-08-12T14:08:10.841Z", + "completedAt": "2026-08-12T14:08:35.678Z", + "requireZeroActiveResourceDelta": true, + "proof": { + "schemaVersion": "braid.tangle-sandbox-braid-stress.v1", + "proofId": "braid-cloud-stress-1786543690842-528499-25488059", + "config": { + "endpoint": "https://sandbox.tangle.tools", + "connectionId": "connection-live-tangle-sandbox", + "connectionKind": "tangle-sandbox", + "credentialConfigured": true, + "model": "tangle-router/glm-5.2", + "runner": "opencode", + "lifecycle": "retained", + "idleTtlSeconds": 300 + }, + "timing": { + "totalMs": 24835.949061000003 + }, + "status": "failed", + "cleanup": { + "exactResource": false, + "activeResourceDelta": 0, + "activeResourceDeltaRequired": true, + "usageObservationComplete": true, + "usageDelta": { + "activeSandboxes": 0, + "totalSandboxes": 1, + "computeMinutes": 0, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "unknownFields": [] + } + }, + "usage": [ + { + "phase": "before", + "status": "observed", + "value": { + "computeMinutes": 906, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9239, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + } + }, + { + "phase": "after", + "status": "observed", + "value": { + "computeMinutes": 906, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9240, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + } + } + ], + "accountIdentities": [ + { + "phase": "before", + "status": "observed", + "value": { + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46", + "billingDelegationAuthorized": false + } + }, + { + "phase": "after", + "status": "observed", + "value": { + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46", + "billingDelegationAuthorized": false + } + } + ], + "accountIdentityConsistency": { + "stable": true, + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46" + }, + "workspaceVerification": null, + "progress": { + "firstRunId": "run-54929ef5-640b-4a63-95af-3526dc967a67" + }, + "failure": { + "name": "MissingIntegrationError", + "message": "Braid run run-54929ef5-640b-4a63-95af-3526dc967a67 became unknown before exposing exact provider identity", + "code": "BRAID_LIVE_INTEGRATION_MISSING", + "details": { + "runId": "run-54929ef5-640b-4a63-95af-3526dc967a67", + "status": "unknown", + "required": "exposing exact provider identity" + } + }, + "diagnostics": { + "firstProcess": { + "responseCount": 10, + "run": { + "id": "run-54929ef5-640b-4a63-95af-3526dc967a67", + "operationId": "op-braid-cloud-stress-1786543690842-528499-25488059-turn-1", + "status": "unknown", + "providerSessionId": "session-braid-run-54929ef5-640b-4a63-95af-3526dc967a67", + "materializationDigest": "7b7c74a29b08f3dd1beb3ce62af2c4f001d5bcc6ec3f6060acd513dcc65ea312", + "observations": { + "localEnvironmentId": null, + "providerEnvironmentId": null, + "environmentRecord": null, + "run": { + "inputTokens": { + "status": "observed", + "value": 0 + }, + "outputTokens": { + "status": "observed", + "value": 0 + }, + "reasoningTokens": { + "status": "missing" + }, + "cacheReadTokens": { + "status": "missing" + }, + "cacheWriteTokens": { + "status": "missing" + }, + "costUsd": { + "status": "missing" + }, + "costStatus": { + "status": "observed", + "value": "unknown" + }, + "usage": { + "status": "missing" + }, + "cost": { + "status": "missing" + }, + "tokensKnown": { + "status": "missing" + }, + "usdKnown": { + "status": "missing" + }, + "usageCompleteness": { + "status": "missing" + }, + "latencyMs": { + "status": "missing" + }, + "durationMs": { + "status": "missing" + }, + "model": { + "status": "observed", + "value": "tangle-router/glm-5.2" + }, + "provider": { + "status": "observed", + "value": "tangle-sandbox" + }, + "runner": { + "status": "observed", + "value": "opencode" + }, + "operationId": { + "status": "observed", + "value": "op-braid-cloud-stress-1786543690842-528499-25488059-turn-1" + }, + "profileSnapshotId": { + "status": "missing" + }, + "connectionId": { + "status": "observed", + "value": "connection-live-tangle-sandbox" + }, + "providerSessionId": { + "status": "observed", + "value": "session-braid-run-54929ef5-640b-4a63-95af-3526dc967a67" + }, + "environmentId": { + "status": "missing" + }, + "replayCursor": { + "status": "missing" + }, + "lastCursor": { + "status": "missing" + }, + "lastProviderSequence": { + "status": "missing" + }, + "eventCount": { + "status": "missing" + }, + "contentBytes": { + "status": "missing" + }, + "contentTruncated": { + "status": "missing" + }, + "missingSequence": { + "status": "missing" + }, + "terminalReason": { + "status": "missing" + }, + "materializationDigest": { + "status": "observed", + "value": "7b7c74a29b08f3dd1beb3ce62af2c4f001d5bcc6ec3f6060acd513dcc65ea312" + }, + "complete": { + "status": "missing" + }, + "status": { + "status": "observed", + "value": "unknown" + }, + "error": { + "status": "missing" + }, + "startedAt": { + "status": "missing" + }, + "updatedAt": { + "status": "missing" + }, + "terminalAt": { + "status": "missing" + }, + "placement": { + "status": "missing" + }, + "resourceSample": { + "status": "missing" + }, + "requestedResources": { + "status": "missing" + }, + "machineId": { + "status": "missing" + }, + "runtimeEndpointHost": { + "status": "missing" + }, + "requestedRegion": { + "status": "missing" + }, + "verifiedRegion": { + "status": "missing" + }, + "storagePersistence": { + "status": "missing" + }, + "gpu": { + "status": "missing" + } + }, + "environment": { + "kind": { + "status": "missing" + }, + "providerEnvironmentId": { + "status": "missing" + }, + "provider": { + "status": "missing" + }, + "name": { + "status": "missing" + }, + "lifecycle": { + "status": "missing" + }, + "lifecycleMode": { + "status": "missing" + }, + "cleanup": { + "status": "missing" + }, + "continuity": { + "status": "missing" + }, + "location": { + "status": "missing" + }, + "region": { + "status": "missing" + }, + "runtimeEndpointHost": { + "status": "missing" + }, + "machineId": { + "status": "missing" + }, + "requestedRegion": { + "status": "missing" + }, + "verifiedRegion": { + "status": "missing" + }, + "storagePersistence": { + "status": "missing" + }, + "requestedResources": { + "status": "missing" + }, + "resourceSample": { + "status": "missing" + }, + "gpu": { + "status": "missing" + }, + "accountUsage": { + "status": "missing" + }, + "unavailableTelemetry": { + "status": "missing" + }, + "placement": { + "status": "missing" + }, + "repository": { + "status": "missing" + }, + "gitRef": { + "status": "missing" + }, + "workingDirectory": { + "status": "missing" + }, + "image": { + "status": "missing" + }, + "createdAt": { + "status": "missing" + }, + "startedAt": { + "status": "missing" + }, + "lastActivityAt": { + "status": "missing" + }, + "expiresAt": { + "status": "missing" + }, + "updatedAt": { + "status": "missing" + } + } + } + }, + "eventTimeline": [ + { + "sequence": 5, + "kind": "draft.changed" + }, + { + "sequence": 6, + "kind": "run.requested", + "status": "admitted" + }, + { + "sequence": 7, + "kind": "effect.upserted" + }, + { + "sequence": 8, + "kind": "run.reconnecting", + "status": "reconnecting" + }, + { + "sequence": 9, + "kind": "run.unknown", + "status": "unknown", + "error": "The provider returned no run record" + }, + { + "sequence": 10, + "kind": "effect.upserted" + } + ] + } + }, + "cleanupFailure": { + "name": "MissingIntegrationError", + "message": "Cleanup found no retained Sandbox for the acknowledged Braid run", + "code": "BRAID_LIVE_INTEGRATION_MISSING", + "details": { + "firstRunId": "run-54929ef5-640b-4a63-95af-3526dc967a67", + "operationId": "op-braid-cloud-stress-1786543690842-528499-25488059-turn-1" + } + }, + "unresolvedIntegrationNeeds": [ + "exposing exact provider identity", + "Braid run run-54929ef5-640b-4a63-95af-3526dc967a67 became unknown before exposing exact provider identity", + "Cleanup found no retained Sandbox for the acknowledged Braid run", + "The live run did not prove the exact retained Braid Sandbox resource identity" + ] + } + } + ] +} diff --git a/artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json b/artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json new file mode 100644 index 0000000..09f58c5 --- /dev/null +++ b/artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json @@ -0,0 +1,214 @@ +{ + "schemaVersion": "braid.tangle-sandbox-braid-soak.v1", + "status": "failed", + "startedAt": "2026-08-12T16:19:56.757Z", + "completedAt": "2026-08-12T16:20:01.454Z", + "requestedRuns": 1, + "attemptedRuns": 1, + "concurrency": 1, + "stoppedAfterCanary": true, + "failures": [ + "run 1 did not pass exact proof", + "run 1: status was not passed", + "cloud environment identity was missing", + "Braid binary digest was missing" + ], + "cleanup": { + "exactProofs": 1, + "exactResourcesRemaining": 0, + "resourceProofsUnavailable": 0, + "activeResourceDelta": 0, + "activeResourceDeltas": [ + 0 + ] + }, + "accountUsage": { + "complete": true, + "before": { + "computeMinutes": 1038, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "after": { + "computeMinutes": 1038, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + }, + "delta": { + "activeSandboxes": 0, + "totalSandboxes": 0, + "computeMinutes": 0, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "unknownFields": [] + } + }, + "latency": { + "totalMs": { + "n": 1, + "min": 4695.301084, + "median": 4695.301084, + "p90": 4695.301084, + "max": 4695.301084 + }, + "phases": {} + }, + "sessionSpend": { + "scope": "every unique local run in every cloud proof", + "rows": [], + "tokens": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "input": 0, + "output": 0 + }, + "cost": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "usd": 0 + } + }, + "attempts": [ + { + "index": 0, + "completionSequence": 0, + "startedAt": "2026-08-12T16:19:56.757Z", + "completedAt": "2026-08-12T16:20:01.453Z", + "requireZeroActiveResourceDelta": true, + "proof": { + "schemaVersion": "braid.tangle-sandbox-braid-stress.v1", + "proofId": "braid-cloud-stress-1786551596758-1578834-b3d20f0d", + "config": { + "endpoint": "https://sandbox.tangle.tools", + "connectionId": "connection-live-tangle-sandbox", + "connectionKind": "tangle-sandbox", + "credentialConfigured": true, + "model": "tangle-router/glm-5.2", + "runner": "opencode", + "lifecycle": "retained", + "idleTtlSeconds": 180 + }, + "timing": { + "totalMs": 4695.301084 + }, + "status": "failed", + "cleanup": { + "exactResource": true, + "mode": "exact-owned-resource-set", + "identity": { + "confirmed": true, + "mode": "exact-owned-resource-set", + "matchedCount": 0, + "removedIds": [], + "deletions": [], + "remainingIds": [] + }, + "activeResourceDelta": 0, + "activeResourceDeltaRequired": true, + "usageObservationComplete": true, + "usageDelta": { + "activeSandboxes": 0, + "totalSandboxes": 0, + "computeMinutes": 0, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "unknownFields": [] + } + }, + "usage": [ + { + "phase": "before", + "status": "observed", + "value": { + "computeMinutes": 1038, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + } + }, + { + "phase": "after", + "status": "observed", + "value": { + "computeMinutes": 1038, + "gpuSeconds": 0, + "gpuCostUsd": 0, + "gpuProviderCostUsd": 0, + "activeSandboxes": 4, + "totalSandboxes": 9241, + "periodStart": "2026-08-12T00:00:00.000Z", + "periodEnd": "2026-08-13T00:00:00.000Z" + } + } + ], + "accountIdentities": [ + { + "phase": "before", + "status": "observed", + "value": { + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46", + "billingDelegationAuthorized": false + } + }, + { + "phase": "after", + "status": "observed", + "value": { + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46", + "billingDelegationAuthorized": false + } + } + ], + "accountIdentityConsistency": { + "stable": true, + "identityDigest": "46505ff5225e7d5c46dcafdfa3142412c4746e7b317b166f6bafa7f443923c46" + }, + "workspaceVerification": null, + "progress": {}, + "failure": { + "name": "Error", + "message": "send failed: CONNECTION_UNSUPPORTED: Retained Tangle execution requires provider-backed lookup after an unacknowledged dispatch" + }, + "diagnostics": { + "firstProcess": { + "responseCount": 3, + "errors": [ + { + "requestId": "braid-live-send-fdfb17a8-0796-47a0-8395-612cab7445fe", + "code": "CONNECTION_UNSUPPORTED", + "message": "Retained Tangle execution requires provider-backed lookup after an unacknowledged dispatch" + } + ] + } + }, + "cleanupFailure": { + "name": "MissingIntegrationError", + "message": "The live run did not prove the exact retained Braid Sandbox resource identity", + "code": "BRAID_LIVE_INTEGRATION_MISSING", + "details": { + "proofId": "braid-cloud-stress-1786551596758-1578834-b3d20f0d" + } + }, + "unresolvedIntegrationNeeds": [ + "The live run did not prove the exact retained Braid Sandbox resource identity" + ] + } + } + ] +} diff --git a/artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json b/artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json new file mode 100644 index 0000000..c68070d --- /dev/null +++ b/artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json @@ -0,0 +1,200 @@ +{ + "schemaVersion": "braid.tangle-sandbox-braid-soak.v1", + "status": "failed", + "startedAt": "2026-08-12T15:23:52.398Z", + "completedAt": "2026-08-12T15:23:56.164Z", + "requestedRuns": 1, + "attemptedRuns": 1, + "concurrency": 1, + "stoppedAfterCanary": true, + "failures": [ + "run 1 did not pass exact proof", + "cloud environment identity was missing", + "Braid binary digest was missing", + "Sandbox account identity was missing", + "cohort active-resource usage was unavailable" + ], + "cleanup": { + "exactProofs": 0, + "exactResourcesRemaining": null, + "resourceProofsUnavailable": 1, + "activeResourceDelta": null, + "activeResourceDeltas": [ + null + ] + }, + "accountUsage": { + "complete": false, + "before": null, + "after": null, + "delta": { + "activeSandboxes": null, + "totalSandboxes": null, + "computeMinutes": null, + "gpuSeconds": null, + "gpuCostUsd": null, + "unknownFields": [ + "activeSandboxes", + "totalSandboxes", + "computeMinutes", + "gpuSeconds", + "gpuCostUsd" + ] + } + }, + "latency": { + "totalMs": { + "n": 1, + "min": 3763.776227, + "median": 3763.776227, + "p90": 3763.776227, + "max": 3763.776227 + }, + "phases": {} + }, + "sessionSpend": { + "scope": "every unique local run in every cloud proof", + "rows": [], + "tokens": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "input": 0, + "output": 0 + }, + "cost": { + "observedRuns": 0, + "unavailableRuns": 0, + "missingRuns": 0, + "usd": 0 + } + }, + "attempts": [ + { + "index": 0, + "completionSequence": 0, + "startedAt": "2026-08-12T15:23:52.398Z", + "completedAt": "2026-08-12T15:23:56.163Z", + "requireZeroActiveResourceDelta": true, + "proof": { + "schemaVersion": "braid.tangle-sandbox-braid-stress.v1", + "proofId": "braid-cloud-stress-1786548232400-1267321-26f346e3", + "config": { + "endpoint": "https://sandbox.tangle.tools", + "connectionId": "connection-live-tangle-sandbox", + "connectionKind": "tangle-sandbox", + "credentialConfigured": true, + "model": "tangle-router/glm-5.2", + "runner": "opencode", + "lifecycle": "retained", + "idleTtlSeconds": 300 + }, + "timing": { + "totalMs": 3763.776227 + }, + "status": "failed", + "cleanup": { + "exactResource": false, + "activeResourceDelta": null, + "activeResourceDeltaRequired": true, + "usageObservationComplete": false, + "usageDelta": { + "activeSandboxes": null, + "totalSandboxes": null, + "computeMinutes": null, + "gpuSeconds": null, + "gpuCostUsd": null, + "unknownFields": [ + "activeSandboxes", + "totalSandboxes", + "computeMinutes", + "gpuSeconds", + "gpuCostUsd" + ] + } + }, + "usage": [ + { + "phase": "before", + "status": "unavailable", + "error": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + } + }, + { + "phase": "after", + "status": "unavailable", + "error": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + } + } + ], + "accountIdentities": [ + { + "phase": "before", + "status": "unavailable", + "error": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + } + }, + { + "phase": "after", + "status": "unavailable", + "error": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + } + } + ], + "accountIdentityConsistency": null, + "workspaceVerification": null, + "progress": {}, + "failure": { + "name": "Error", + "message": "send failed: CONNECTION_UNSUPPORTED: The published Tangle provider did not report exact retained-run control" + }, + "diagnostics": { + "firstProcess": { + "responseCount": 3, + "errors": [ + { + "requestId": "braid-live-send-391e963c-e08f-44ab-9e64-29287e561ac7", + "code": "CONNECTION_UNSUPPORTED", + "message": "The published Tangle provider did not report exact retained-run control" + } + ] + } + }, + "cleanupFailure": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + }, + "accountIdentityFailure": { + "name": "MissingIntegrationError", + "message": "Sandbox account identity was unavailable before execution", + "code": "BRAID_LIVE_INTEGRATION_MISSING", + "details": { + "observation": { + "phase": "before", + "error": { + "name": "AuthError", + "message": "Platform key is revoked, expired, or not eligible" + } + } + } + }, + "unresolvedIntegrationNeeds": [ + "Sandbox account identity was unavailable before execution", + "The live run did not prove the exact retained Braid Sandbox resource identity", + "Sandbox usage must expose activeSandboxes before and after cleanup", + "Sandbox usage-before observation failed", + "Sandbox usage-after observation failed", + "Sandbox usage must expose activeSandboxes before execution and after cleanup" + ] + } + } + ] +} diff --git a/docs/01-product-contract.md b/docs/01-product-contract.md index 4c73034..14b0c31 100644 --- a/docs/01-product-contract.md +++ b/docs/01-product-contract.md @@ -44,7 +44,9 @@ They need fast streaming, readable tools and reasoning, precise cancellation, pr ### Tangle cloud user -The cloud user wants the same profile and conversation experience in an isolated Tangle sandbox, with reconnect, workspace operations, checkpoints, environment forks, and placement visibility. +The cloud user wants the same profile and conversation experience in an isolated Tangle sandbox. + +Reconnect, workspace operations, checkpoints, environment forks, and placement appear only when the provider proves them. They need to know what persisted, what moved to the cloud, and the exact state a fork copied. @@ -92,9 +94,10 @@ They need analyses to remain separate from the conversation they analyze unless 1. The user opens the connection picker and chooses a Tangle sandbox connection. 2. Braid validates the profile against reported cloud capabilities. 3. The user confirms workspace source, placement, resource policy, confidentiality, and estimated execution context. -4. The Tangle provider creates or resumes the environment and the runtime starts the run. -5. Braid can detach, reconnect, inspect the workspace, checkpoint it, and create an environment fork when capabilities permit. -6. The branch graph records which environment and checkpoint back each branch. +4. The Tangle provider creates one ephemeral environment by default and the runtime starts the run. +5. An explicit retained connection can recover an environment only when exact control and provider-backed lookup are available. +6. Braid can detach, reconnect, inspect the workspace, checkpoint it, and create an environment fork when capabilities permit. +7. The branch graph records which environment and checkpoint back each branch. ### Change runner without changing agent @@ -243,14 +246,14 @@ The interface never calls a connection an agent, never calls a runner a model, a | ID | Required outcome | | --- | --- | | PR-01 | A new user completes one real CLI Bridge turn from an existing profile without editing a configuration file. | -| PR-02 | The same profile completes one real Tangle sandbox turn and the interface explains the changed placement. | +| PR-02 | The same profile completes one real ephemeral Tangle sandbox turn, explains placement, and confirms environment deletion. | | PR-03 | The user changes runner for a new branch without changing the stored profile and can inspect both run snapshots. | -| PR-04 | A disconnected durable run resumes from its cursor with no missing or duplicated displayed event. | +| PR-04 | A provider-reported durable run resumes from its cursor with no missing or duplicated displayed event. | | PR-05 | A permission request reaches the terminal, receives a scoped response, and the waiting run continues through the shared contract. | | PR-06 | Conversation-only and environment forks display different, accurate copy semantics before confirmation. | | PR-07 | `/ask` creates a separate cited analysis node and does not add a message to the analyzed branch. | | PR-08 | A live runtime worker can be inspected, steered, and cancelled through a runtime API that confirms the effect. | -| PR-09 | Restarting Braid reconstructs the same conversation graph, interaction decisions, and run bindings from the local journal. | +| PR-09 | Restarting Braid reconstructs the local graph, decisions, and saved bindings without claiming unavailable remote control. | | PR-10 | A keyboard-only user completes every primary journey at 80×24 and a narrow 40×12 terminal exposes no unreachable action. | | PR-11 | Headless commands reproduce the same state transitions and validation results as the terminal interface. | | PR-12 | One release manifest proves all required deterministic, terminal, live, visual, semantic, security, install, and performance checks against one immutable Braid build. | diff --git a/docs/03-architecture.md b/docs/03-architecture.md index 10ca07b..ae73bba 100644 --- a/docs/03-architecture.md +++ b/docs/03-architecture.md @@ -275,6 +275,32 @@ The production SQLite implementation makes `appendEffect` durable before it retu The execution adapter must be thin enough that contract tests can run directly against both CLI Bridge and Tangle providers. +### Retained execution composition + +`RetainedExecutionPort` owns the provider-neutral lifecycle for admission, start, event reading, cursor tracking, detach, reconnect, status, and cancellation. + +It stores only process-local handles and exact provider bindings; the Braid journal remains the durable authority. + +Each provider supplies a small plan that resolves one current provider, maps status and result records, and delegates start, reconnect, and cancel to `agent-runtime`. + +The CLI Bridge plan adds Bridge discovery because that service can recover a run before Braid has its exact reference. + +The Tangle plan accepts only provider-backed lookup that returns the exact run reference from deterministic request keys. + +The production path stays unavailable when that lookup is absent. + +The Tangle resolver validates the selected `AgentProfile`, connection, runner, model, client methods, and retained capabilities without creating a sandbox. + +The Tangle policy module owns only deterministic environment tags and bounded idle expiry. + +The Tangle run module owns only Runtime calls and never guesses whether an idempotent create returned a new or existing environment. + +An ambiguous dispatch failure can be recovered by lookup, but cleanup still waits for a creation receipt that proves ownership. + +No retained module runs an agent loop, parses runner output, or becomes a second provider implementation. + +This split prevents one execution object from accumulating provider setup, durable state, product rules, cloud cleanup, event projection, and terminal behavior. + ## Event path ```mermaid diff --git a/docs/04-runtime-contracts.md b/docs/04-runtime-contracts.md index 19eeea2..bd720e2 100644 --- a/docs/04-runtime-contracts.md +++ b/docs/04-runtime-contracts.md @@ -15,23 +15,25 @@ The following published versions were queried from npm and their installed decla | Package | Installed version | Braid boundary | | --- | ---: | --- | | [`@tangle-network/agent-interface`](https://github.com/tangle-network/agent-sdk/tree/main/packages/agent-interface) | `0.47.0` | Canonical profile, capabilities, environment, stream, portable context, and interaction contracts | -| [`@tangle-network/agent-runtime`](https://github.com/tangle-network/agent-runtime) | `0.132.9` | Sole execution layer; exact executor, retained-run, environment-provider, and terminal-monitor exports | -| [`@tangle-network/agent-eval`](https://github.com/tangle-network/agent-eval) | `0.145.1` | Run records, judges, trace analysts, comparisons, and feedback trajectories | +| [`@tangle-network/agent-runtime`](https://github.com/tangle-network/agent-runtime) | `0.132.12` | Sole execution layer; exact executor, retained-run, environment-provider, and terminal-monitor exports | +| [`@tangle-network/agent-eval`](https://github.com/tangle-network/agent-eval) | `0.145.3` | Run records, judges, trace analysts, comparisons, and feedback trajectories | | `@tangle-network/agent-provider-cli-bridge` | `0.6.0` | CLI Bridge environment adapter with live streaming, replay, retry-safe turns, and explicit cancel | | `@tangle-network/agent-provider-tangle` | `0.6.3` | Tangle environment adapter over the sandbox client | -| `@tangle-network/sandbox` | `0.19.6` | Tangle cloud client used by the provider | +| `@tangle-network/sandbox` | `0.21.1` | Tangle cloud client used by the provider | -The installed runtime publishes `agent-eval >=0.145.0 <0.146.0`, `agent-interface >=0.46.1 <0.47.0`, and optional `sandbox >=0.19.4 <0.20.0` as peer ranges. +The installed runtime publishes `agent-eval >=0.145.2 <0.146.0`, `agent-interface >=0.47.0 <0.48.0`, and `sandbox >=0.21.1 <0.22.0` as peer ranges. -Braid exercises runtime `0.132.9` with interface `0.47.0`, eval `0.145.1`, CLI Bridge adapter `0.6.0`, Tangle adapter `0.6.3`, and sandbox `0.19.6`. +Braid exercises runtime `0.132.12` with interface `0.47.0`, eval `0.145.3`, CLI Bridge adapter `0.6.0`, Tangle adapter `0.6.3`, and sandbox `0.21.1`. The lockfile pins the registry integrity for every installed package. -Runtime `0.132.9` does not yet accept interface `0.47.x` in its published peer range. +`pnpm peers check` reports no peer dependency issues for this package graph. -The workspace override resolves every transitive `agent-interface` dependency to `0.47.0`; the package tests prove one compatible contract graph. +The workspace override resolves every transitive `agent-interface` dependency to `0.47.0` and prevents duplicate contract graphs. -[Agent-runtime issue 803](https://github.com/tangle-network/agent-runtime/issues/803) tracks the remaining published peer-range mismatch. +[Agent-runtime issue 803](https://github.com/tangle-network/agent-runtime/issues/803) records the interface peer mismatch fixed in Runtime `0.132.11`. + +[Agent-runtime issue 737](https://github.com/tangle-network/agent-runtime/issues/737) records the Sandbox peer mismatch fixed in Runtime `0.132.11`. Braid imports only the canonical root `agent-interface` entry point behind two local modules. @@ -140,7 +142,7 @@ A partial or inconsistent positive-input split must set `cacheBreakdownKnown: fa Braid rejects unknown fields and inconsistent totals under token-bearing record names. -Runtime `0.132.9` currently buffers executor output until the executor settles. +Runtime `0.132.12` currently buffers executor output until the executor settles. Braid therefore receives terminal text, tool calls, usage, and result evidence but cannot render provider text deltas live through this path. @@ -166,7 +168,7 @@ The capability is a typed Runtime-executor tag, not a provider-specific callback When the tag is present, `/cancel` calls the public Runtime `Executor.teardown('infinity')` operation and waits for its result before committing control state. -The installed Runtime `0.132.9` bridge executor implements that operation by posting `POST /v1/runs/:id/cancel` and waiting for a terminal bridge snapshot. +The installed Runtime `0.132.12` bridge executor implements that operation by posting `POST /v1/runs/:id/cancel` and waiting for a terminal bridge snapshot. `destroyed: true` becomes an accepted cancellation, while `destroyed: false`, a thrown error, or a control deadline becomes unknown. @@ -188,7 +190,7 @@ The upstream Runtime change required to expand this support is recorded below wi Title: `Expose typed, signal-aware provider cancellation acknowledgement from Executor` -Runtime `0.132.9` exposes `Executor.teardown(grace): Promise<{ destroyed: boolean }>` in its published declaration bundle. +Runtime `0.132.12` exposes `Executor.teardown(grace): Promise<{ destroyed: boolean }>` in its published declaration bundle. The bridge executor's `teardown('infinity')` posts `POST /v1/runs/:id/cancel` and waits for a terminal bridge snapshot in `dist/supervisor-BI6Z-8Yi.js:6973-6977,7529-7570`. @@ -331,15 +333,21 @@ It exposes no generalized interaction response. ## Existing Tangle provider contract -The published `@tangle-network/agent-provider-tangle@0.6.2` wraps `@tangle-network/sandbox` as an `AgentEnvironmentProvider`. +The published `@tangle-network/agent-provider-tangle@0.6.3` wraps `@tangle-network/sandbox` as an `AgentEnvironmentProvider`. Its default document is an upper bound, not a claim about one client or deployment. -It reports canonical profile dimensions, live and replay streaming, detach, turn idempotency, workspace read/write/exec/upload/download, and optional placement. +Its default capability document reports canonical profile dimensions, live and replay streaming, detach, turn idempotency, workspace read/write/exec/upload/download, and optional placement. + +Its default document reports native continuation, session listing, session messages, workspace git, checkpoint, fork, usage, and confidentiality as unavailable. + +Version `0.6.3` accepts an explicit capability declaration and narrows it against the concrete client and environment methods. + +Braid does not inject positive retained capabilities into the provider. -It reports native continuation, session listing, session messages, workspace git, checkpoint, fork, usage, and confidentiality as unavailable. +It requires client `get`, exact control, replay, detach, turn idempotency, retry-safe cancellation, and provider-backed dispatch lookup. -The adapter narrows placement against the client and narrows environment methods after creation. +The current default provider report and methods do not satisfy that requirement. The adapter exposes environment stream and dispatch, provider sessions, workspace methods, refresh, and destroy only when the sandbox instance implements them. @@ -375,21 +383,44 @@ Direct inference records the service endpoint host and unavailable Router accoun Tangle sandbox records provider sandbox identity, lifecycle, cleanup, continuity, requested resources, public endpoint host, placement, cgroup CPU and RAM use, GPU lease billing, and sandbox account usage when available. -The current Tangle sandbox path is deleted after each turn and rejects session continuation. +New Tangle sandbox connections default to ephemeral lifecycle. + +That path deletes its environment after one turn and rejects session continuation. + +An explicit retained lifecycle requires an idle limit from 60 through 604,800 seconds. + +The resolver rejects retained execution before resource creation unless exact control and dispatch lookup are both available. + +The retained adapter composes Runtime `startRetainedRun`, `reconnectRetainedRun`, and exact native cancellation after that check passes. + +Its saved control reference contains provider, environment, session, execution, provider run, and request-digest identity. + +Native follow-up turns remain disabled because the current provider does not prove a matching context boundary. + +An ambiguous retained start failure does not destroy its environment because an idempotent create can return a pre-existing workspace. + +Safe compensation requires a provider-issued receipt that distinguishes a new create from an idempotent replay. + +If either Tangle path requests approval, an answer, or a plan decision, Braid fails the turn with an unsupported-interaction explanation. -If that ephemeral path requests approval, an answer, or a plan decision, Braid fails the turn with an unsupported-interaction explanation. +Braid does not display a resumable interaction until the provider exposes a durable response operation. -Braid does not display a resumable interaction after its environment is deleted. +Braid does not admit retained Tangle execution without lookup for the crash window before its exact reference commits. -The platform supports retained sandboxes, but Runtime does not yet expose safe retained identity and recovery to Braid. +The local retained test proves that lookup recovers and cancels a run after a simulated process loss in that window. The observation record never contains API keys, bearer tokens, SSH credentials, secret values, credential-bearing URLs, Docker host strings, or internal listener addresses. The following upstream issues own missing shared contracts: -- [Runtime issue 799](https://github.com/tangle-network/agent-runtime/issues/799) requires exact cleanup when retained dispatch fails after environment creation. +- [Runtime issue 799](https://github.com/tangle-network/agent-runtime/issues/799) requires a creation receipt and exact cleanup when retained dispatch fails. - [Runtime issue 800](https://github.com/tangle-network/agent-runtime/issues/800) requires crash-safe exact run admission or deterministic discovery. - [Agent SDK issue 146](https://github.com/tangle-network/agent-sdk/issues/146) requires retained Tangle control, recovery, interactions, and workspace branching. +- [Sandbox issue 5249](https://github.com/tangle-network/agent-dev-container/issues/5249) requires a missing completed turn to return a cache miss instead of throwing. +- [Sandbox issue 5251](https://github.com/tangle-network/agent-dev-container/issues/5251) requires the exact retained-run revision to reach staging and production. +- [Sandbox issue 5277](https://github.com/tangle-network/agent-dev-container/issues/5277) requires valid Sandbox keys to authorize internal model-key provisioning. +- [Sandbox issue 5278](https://github.com/tangle-network/agent-dev-container/issues/5278) requires the npm `latest` tag to satisfy current Runtime peers. +- [Runtime issue 808](https://github.com/tangle-network/agent-runtime/issues/808) requires permanent provisioning rejections to fail without the ten-minute retry. - [Runtime issue 763](https://github.com/tangle-network/agent-runtime/issues/763) requests one stable execution tree with complete usage provenance. - [Agent SDK issue 136](https://github.com/tangle-network/agent-sdk/issues/136) requests normalized provider observations and account usage. - [Sandbox issue 5076](https://github.com/tangle-network/agent-dev-container/issues/5076) requests resolved placement, effective resources, and per-sandbox billing. @@ -416,7 +447,7 @@ For CLI Bridge, Braid uses Runtime's `startRetainedRun` with the selected provid Braid returns normalized output, measured token usage, priced cost, terminal status, and finite redacted execution evidence to `agent-eval`. -The callback rejects multimodal and request-level thinking controls because runtime `0.132.9` does not expose those fields on this exact turn input. +The callback rejects multimodal and request-level thinking controls because runtime `0.132.12` does not expose those fields on this exact turn input. Reasoning remains an `AgentProfile` setting, and unsupported callback shapes fail before a provider call rather than being silently dropped. @@ -662,7 +693,7 @@ The release manifest records package names, versions, integrity hashes, source c | UP-06 | CLI Bridge removes unconditional ACP and OpenCode auto-approval from interactive mode and proves explicit automation remains opt-in and profile-scoped. | | UP-07 | CLI Bridge response retry returns the same acknowledgement for the same operation and a conflict for a different response after resolution. | | UP-08 | The published CLI Bridge provider proves replay, detach, recreateable run control, interaction response, status, and terminal cancel against a real bridge server. | -| UP-09 | The published Tangle provider proves replayed interaction request, post-reconnect response, run cancel, checkpoint, and environment fork against a real sandbox deployment. | +| UP-09 | The published Tangle provider proves exact retained create, dispatch lookup, replay, interaction response, cancel retry, and cleanup against a real deployment. | | UP-10 | Runtime supervisor tests prove watch, typed steer, and typed cancel effects in-process and after control-client reconnect; no production path relies on an unread request file. | | UP-11 | Published shared package versions install together with no ignored peer mismatch and Braid's contract suite passes against tarballs rather than workspace links. | | UP-12 | Capability conformance tests deliberately disable each capability and prove Braid does not expose or call the corresponding action. | diff --git a/docs/05-profiles-and-connections.md b/docs/05-profiles-and-connections.md index 403cb24..b53ea3d 100644 --- a/docs/05-profiles-and-connections.md +++ b/docs/05-profiles-and-connections.md @@ -200,6 +200,12 @@ interface ConnectionRecord { updatedAt: string lastHealth?: ConnectionHealth } + +interface ConnectionTransportOptions { + transport?: 'http' | 'https' | 'sse' + lifecycle?: 'ephemeral' | 'retained' + idleTtlSeconds?: number +} ``` This record is a Braid storage type, not a replacement for provider configuration types. @@ -264,7 +270,29 @@ The setup preview shows repository, ref, image or environment, CPU, memory, disk Connection health proves authentication and provider reachability but does not create a sandbox. -A separate one-environment smoke proves create, prompt, replay, workspace operations, checkpoint, fork, and destroy in release verification. +New setup records default to ephemeral lifecycle. + +A connection configuration can set retained lifecycle with an idle limit from 60 through 604,800 seconds. + +An idle limit is rejected for an ephemeral connection, and a retained connection without one is rejected. + +The ephemeral path deletes its environment after one turn. + +The retained resolver performs no create operation. + +It rejects the connection before resource creation unless exact control and provider-backed dispatch lookup are both available. + +After that check passes, the first admitted turn creates one environment with a deterministic Braid name, metadata owner, provider session, and retry key. + +Braid records the exact provider run reference before replay, continuation, cancellation, or cleanup can claim the environment. + +A fresh Braid process can discover, replay, and cancel that exact run before or after the reference commits. + +Native follow-up turns remain unavailable until the provider proves a matching context boundary. + +The retained release stress requires create, active-process disconnect, fresh-process replay, hidden workspace continuity, retry-safe cancellation, exact destroy, and zero account-level active-resource delta. + +Checkpoint, environment fork, and interaction response remain disabled until their provider methods and live checks pass. Attestation status is displayed only after cryptographic verification through the current confidential-execution contract. diff --git a/docs/06-conversations-forks-and-analysis.md b/docs/06-conversations-forks-and-analysis.md index 6af9d0d..aede0cb 100644 --- a/docs/06-conversations-forks-and-analysis.md +++ b/docs/06-conversations-forks-and-analysis.md @@ -421,9 +421,9 @@ Conflicting rules fail closed and require user response. | CF-05 | Branch, clone, full fork, cross-runner handoff, retry, checkpoint, and environment fork each produce the specified identifiers and graph edges. | | CF-06 | A real cloud workspace fork proves source checkpoint, destination environment, independent file mutation, and unchanged source workspace. | | CF-07 | Pending interactions, queued future turns, opaque provider state, and unselected analyses never enter inherited context. | -| CF-08 | Restart during each external fork step reconciles by idempotency key and request digest without duplicate branch, checkpoint, environment, session, or context transfer, and confirms cleanup of recovered remote resources. | +| CF-08 | Restart during each external step reconciles by idempotency key and request digest without duplicate work, including dispatch before exact-reference commit, and confirms cleanup of recovered resources. | | CF-09 | Export and import preserve graph checksum, receipts, citations, completeness, and redaction while leaving external controls disabled until reconciliation. | -| CF-10 | A missing provider session produces a fresh-session handoff or honest unavailable state and never a false native resume label. | +| CF-10 | A missing provider session or exact control reference produces a fresh-session handoff or honest unavailable state and never a false resume label. | | AN-01 | `/ask` freezes one immutable source digest and late events cannot change the running or completed analysis. | | AN-02 | `/ask` creates no message or context mutation in the source branch before explicit promotion. | | AN-03 | Every cited finding resolves to the frozen event or span range and an invalid citation is deterministically rejected. | diff --git a/docs/08-verification.md b/docs/08-verification.md index 6e7716f..b62c07e 100644 --- a/docs/08-verification.md +++ b/docs/08-verification.md @@ -365,8 +365,8 @@ The published package is downloaded from the registry after publication and its | LIVE-04 | CLI Bridge restart | Run state becomes honestly unknown or recovers according to retained state; Braid never labels it cancelled or resubmits unsafely | | LIVE-05 | Every advertised interactive bridge runner | Common conformance flow at a pinned minimum runner version; failures remove the interactive capability claim | | LIVE-06 | Tangle inference | Real profile-backed inference route, streaming, usage, cancellation, and immutable receipt | -| LIVE-07 | Tangle sandbox | Environment create, profile validation, turn, replay after client restart, workspace read/write/exec/git, run cancel, and retained environment | -| LIVE-08 | Tangle interaction | Replayed cloud interaction remains answerable after Braid reconnect and the session continues from the response | +| LIVE-07 | Tangle sandbox | Ephemeral create, turn, observation, and deletion; retained exact lookup, forced process loss, replay, cancel retry, and confirmed cleanup | +| LIVE-08 | Tangle interaction | A retained cloud interaction remains answerable after Braid reconnect and continues once from the acknowledged response | | LIVE-09 | Tangle workspace fork | Checkpoint, destination fork, independent destination file change, unchanged source file, and explicit cleanup of both environments | | LIVE-10 | Confidential Tangle path | Requested placement remains unverified until valid attestation is checked; negative nonce and measurement tests fail | | LIVE-11 | Runtime supervisor | Real root and worker stream, spend and status update, typed steering effect, typed cancellation effect, and reconnectable control | @@ -386,6 +386,8 @@ The command stops scheduling new proofs after any failure. Each proof creates one retained cloud environment through Braid and records the exact six-field cloud identity. +When the provider does not report the required exact-control contract, the canary must fail before resource creation and the cohort must not start. + The proof kills the first Braid process after a committed provider cursor. A fresh Braid process must reconnect to the same cloud execution without duplicate visible provider events. @@ -432,13 +434,91 @@ The artifact is `artifacts/verification/live/tangle-sandbox-braid-execution-stre Its SHA-256 is `1a38a26e97917073ef760525f7b18abbcca43fcf6faba54cf862e55e4886693c`. -This proves production cloud execution, observation, concurrency, and cleanup only. +This proves production ephemeral cloud execution, observation, concurrency, and cleanup only. + +The retained implementation passes its local lifecycle, restart, CLI Bridge regression, and error-observability tests. + +The post-hardening retained canary used a fresh bounded Sandbox key and stopped before allocation in 4.695 seconds. + +Braid rejected the run because the published provider does not supply lookup after an unacknowledged dispatch. + +Account identity stayed stable, active Sandboxes stayed at four, total Sandboxes stayed at 9,241, and every metered usage delta was zero. + +The exact owner-tag query matched zero resources before and after cleanup. + +The artifact is `artifacts/verification/live/tangle-sandbox-braid-retained-post-hardening-production-20260812.json`. + +Its SHA-256 is `625f3bf9f90bec707485592e5cab2226cefd8cdc6020a82c6d98e4bec2064fe9`. + +This is admissible failure evidence for safe retained admission, but it is not a passing retained-run proof. + +An earlier capability-injected production diagnostic canary stopped after its first attempt in 24.836 seconds. + +The attempt increased total Sandboxes by one, observed zero compute-minute change, and returned active Sandbox count from four to four. + +The owner-tag lookup found no Braid environments afterward. + +Exact deletion remained unconfirmed because Braid never received the resource identity. + +The direct start trace first found that Sandbox SDK `0.19.6` throws for a missing-turn `404` whose retry meaning is not machine-readable. + +Braid now fails closed on every numeric `404` instead of converting an ambiguous response into a cache miss. + +The next direct start reached production and failed because the deployed sidecar rejected `runControlRef` as an unknown field. + +Production serves ADC revision `7437945e19bb`, and staging serves `955e6e767e77`; neither contains exact retained-run support from `f3e09d3bf`. + +The failed secret-free canary artifact is `artifacts/verification/live/tangle-sandbox-braid-retained-canary-production-20260812.json`. + +Its SHA-256 is `c411474b31e51643e724cace10e58cba6695619e67d52640bc3bd5515650a550`. + +The artifact predates the fail-closed provider capability check and is diagnostic evidence, not release acceptance for the current implementation. + +An older preflight rejected retained execution before dispatch in 3.764 seconds because the published provider lacked the required retained contract. + +The vault Sandbox key was revoked, so account usage and cleanup queries were unavailable and that preflight cannot prove a zero-resource delta. + +Its diagnostic artifact is `artifacts/verification/live/tangle-sandbox-braid-retained-safe-preflight-production-20260812.json`. + +Its SHA-256 is `840a6234c26b7efd6f9478fb27bca753ff2726a2bb348d41480dc7ee27bd0cb9`. + +The authenticated post-hardening canary supersedes this older credential-limited preflight. + +A later ephemeral Braid canary selected Sandbox, OpenCode, and `tangle-router/glm-5.2`, but no environment reached creation. + +The canary timed out after 182.581 seconds while Runtime kept the run in `streaming`. + +Active and total Sandbox counts stayed unchanged, and the exact cleanup query found zero owned resources. + +The account-wide compute counter increased by four minutes during the wait, so that delta cannot be attributed to this zero-allocation run. + +Its artifact is `artifacts/verification/live/tangle-sandbox-braid-execution-canary-post-hardening-production-20260812.json`. + +Its SHA-256 is `78c22b68221606c9acd6abd6f79094e58774c8a4863fdc7aec7e1f9258011b2c`. + +A direct Sandbox SDK request with the same profile failed in 2.177 seconds with outer HTTP 400 `CONFIG_ERROR` and nested HTTP 403. + +The nested service message was `Service "sandbox" is not authorized for this endpoint`. + +The same bounded key authenticated Sandbox usage and the Router model list before revocation. + +[Sandbox issue 5277](https://github.com/tangle-network/agent-dev-container/issues/5277) owns this production authorization failure. + +Runtime `0.132.12` still retries this permanent error because its text contains `provision failed`. + +[Runtime issue 808](https://github.com/tangle-network/agent-runtime/issues/808) owns the fail-fast correction. + +Sandbox npm `latest` resolves to `0.21.0`, although Runtime `0.132.12` requires Sandbox `>=0.21.1 <0.22.0`. + +Braid pins `0.21.1`, and [Sandbox issue 5278](https://github.com/tangle-network/agent-dev-container/issues/5278) owns the incorrect tag. + +[Sandbox issue 5249](https://github.com/tangle-network/agent-dev-container/issues/5249) owns the SDK response mismatch. -It does not pass `LIVE-07` because the current provider cannot recover an exact retained run after a Braid process crash. +[Sandbox issue 5251](https://github.com/tangle-network/agent-dev-container/issues/5251) owns the blocked platform rollout. -Production and staging durability canaries both failed closed and left zero owned resources. +`LIVE-07` remains blocked until cloud creation works, retained lookup ships, and the default three-proof cohort passes. -[Runtime issue 800](https://github.com/tangle-network/agent-runtime/issues/800) and [Agent SDK issue 146](https://github.com/tangle-network/agent-sdk/issues/146) own the missing shared recovery path. +[Runtime issue 800](https://github.com/tangle-network/agent-runtime/issues/800) owns the shared lookup contract; Braid keeps retained production admission off until a provider supplies it. ## Runner conformance diff --git a/docs/09-delivery-plan.md b/docs/09-delivery-plan.md index 8730948..bf73d62 100644 --- a/docs/09-delivery-plan.md +++ b/docs/09-delivery-plan.md @@ -181,6 +181,7 @@ Closing a Braid interface and explicitly cancelling a run must produce different - Carry canonical interaction requests in sandbox session events and accept idempotent typed responses. - Preserve interaction state and replay across Braid disconnect and reconnect. +- Recover an exact retained dispatch by deterministic keys after caller death and before local reference commit. - Expose run cancellation separately from environment destruction. - Validate inline profiles and return effective capability, placement, session, usage, and confidentiality evidence. - Prove workspace read, write, exec, Git, upload, download, checkpoint, and fork methods against current deployment support. diff --git a/docs/10-upstream-strategy.md b/docs/10-upstream-strategy.md index 6848a42..8c96e9e 100644 --- a/docs/10-upstream-strategy.md +++ b/docs/10-upstream-strategy.md @@ -37,7 +37,7 @@ The deciding question was how much source can be reused while preserving `AgentP | [OpenCode terminal packages](https://github.com/anomalyco/opencode/tree/dev/packages/tui) | `3a90639cb57619a21e59f544b3e8d23ffed56f48`; npm `1.18.16` | MIT | 50,341 lines across the TUI and run command | OpenTUI alternative and workflow reference | | [Codex terminal application](https://github.com/openai/codex/tree/main/codex-rs/tui) | repository `2cc9dbb`; TUI `9742cc8ed5def37a4575263733f70a01ca22047b`; CLI `0.147.0` | Apache-2.0 | 244,714 Rust lines | Composer, worker navigation, status, and snapshot-test reference | | [Hermes Agent terminal app](https://github.com/NousResearch/hermes-agent/tree/main/ui-tui) | `697f2896bf948731eb6fcb93caa7264478590843` | MIT | No size claim used | Client/runtime and workflow reference | -| [`agent-runtime` terminal monitor](https://github.com/tangle-network/agent-runtime/tree/main/src/tui) | repository `9c18cb48`; npm `0.132.9` | Project license | No refreshed size claim | Runtime-owned supervisor source, not app base | +| [`agent-runtime` terminal monitor](https://github.com/tangle-network/agent-runtime/tree/main/src/tui) | repository `9c18cb48`; npm `0.132.11` | Project license | No refreshed size claim | Runtime-owned supervisor source, not app base | The count commands selected the named TypeScript or Rust files from sparse clones and used `wc -l`. @@ -165,7 +165,7 @@ Braid uses Hermes as a workflow comparison and does not port its runtime protoco ## Runtime monitor -`agent-runtime` package version `0.132.9` exports a diagnostic terminal module and `agent-runtime-top` binary. +`agent-runtime` package version `0.132.12` exports a diagnostic terminal module and `agent-runtime-top` binary. The module understands runtime-owned supervisor files and shows worker state, spend, tokens, latency, logs, steering, shell action, and cancellation controls. diff --git a/docs/decisions/002-runtime-boundary.md b/docs/decisions/002-runtime-boundary.md index 1d367e2..81e243c 100644 --- a/docs/decisions/002-runtime-boundary.md +++ b/docs/decisions/002-runtime-boundary.md @@ -30,6 +30,8 @@ The runtime remains responsible for lifecycle, execution admission, event normal Providers remain responsible for native sessions, transport replay, workspaces, and profile materialization. +Providers also own exact lookup for remote work accepted before Braid commits its control reference. + ## Consequences Interactive permission prompts work the same way for local and cloud runs. diff --git a/docs/decisions/003-local-event-journal.md b/docs/decisions/003-local-event-journal.md index cb41bf0..a8d7300 100644 --- a/docs/decisions/003-local-event-journal.md +++ b/docs/decisions/003-local-event-journal.md @@ -22,6 +22,10 @@ The journal is authoritative for Braid's product graph and user decisions. The provider remains authoritative for the live process, native session, cloud environment, and provider-specific replay cursor. +Pre-dispatch journaling protects local intent but does not prove that remote work can be recovered. + +Braid enables remote retention only when the provider can look up the exact accepted work from durable request keys. + Every ingested local event uses a unique `(run_id, event_id)` key. Provider event identity is stored separately because a provider's event identifier has provider-session scope and must not become Braid's global identity. diff --git a/docs/decisions/004-application-effect-coordination.md b/docs/decisions/004-application-effect-coordination.md index 8ca8d68..c5b7fba 100644 --- a/docs/decisions/004-application-effect-coordination.md +++ b/docs/decisions/004-application-effect-coordination.md @@ -34,6 +34,10 @@ A previously pending operation is resolved only through an explicit reconciliati Without reconciliation evidence the coordinator leaves it pending and never guesses that repeating the mutation is safe. +A pending local effect is not a remote control reference. + +Remote retention requires provider lookup for success that occurred before the exact reference committed. + `JournalPort` and `EffectStoragePort` are application seams rather than new provider protocols. ## Consequences diff --git a/package.json b/package.json index 4248895..94b883c 100644 --- a/package.json +++ b/package.json @@ -99,12 +99,12 @@ "@dataiku/uv": "0.12.0", "@earendil-works/pi-tui": "0.84.1", "@napi-rs/keyring": "1.3.0", - "@tangle-network/agent-eval": "0.145.1", + "@tangle-network/agent-eval": "0.145.3", "@tangle-network/agent-interface": "0.47.0", "@tangle-network/agent-provider-cli-bridge": "0.6.0", "@tangle-network/agent-provider-tangle": "0.6.3", - "@tangle-network/agent-runtime": "0.132.10", - "@tangle-network/sandbox": "0.19.6", + "@tangle-network/agent-runtime": "0.132.12", + "@tangle-network/sandbox": "0.21.1", "better-sqlite3-multiple-ciphers": "13.0.3", "chalk": "6.0.0", "koffi": "3.1.4" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67cf064..5d76ce2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,8 +23,8 @@ importers: specifier: 1.3.0 version: 1.3.0 '@tangle-network/agent-eval': - specifier: 0.145.1 - version: 0.145.1 + specifier: 0.145.3 + version: 0.145.3 '@tangle-network/agent-interface': specifier: 0.47.0 version: 0.47.0 @@ -33,13 +33,13 @@ importers: version: 0.6.0 '@tangle-network/agent-provider-tangle': specifier: 0.6.3 - version: 0.6.3(@tangle-network/sandbox@0.19.6) + version: 0.6.3(@tangle-network/sandbox@0.21.1) '@tangle-network/agent-runtime': - specifier: 0.132.10 - version: 0.132.10(@tangle-network/agent-eval@0.145.1)(@tangle-network/agent-interface@0.47.0)(@tangle-network/sandbox@0.19.6) + specifier: 0.132.12 + version: 0.132.12(@tangle-network/agent-eval@0.145.3)(@tangle-network/agent-interface@0.47.0)(@tangle-network/sandbox@0.21.1) '@tangle-network/sandbox': - specifier: 0.19.6 - version: 0.19.6 + specifier: 0.21.1 + version: 0.21.1 better-sqlite3-multiple-ciphers: specifier: 13.0.3 version: 13.0.3 @@ -504,30 +504,27 @@ packages: resolution: {integrity: sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==} engines: {node: ^14.21.3 || >=16} - '@tangle-network/agent-core@0.5.4': - resolution: {integrity: sha512-k6gYv3BlagkfuWrGyTJH6mKUBgsLY6TXxizACqt0QF8a1/5uqy0UYc6R2Wo9nqQVJuaRxDnoiRf8YtsRVqA75g==} + '@tangle-network/agent-core@0.6.1': + resolution: {integrity: sha512-vPpxnSmXpXA9NBjmbOzEnocMJBdskMTrueCepJnztQsO3Z7CKpPQShcJbA5XRv4KyH3KPF284vgMI5lojErvsw==} - '@tangle-network/agent-core@0.6.0': - resolution: {integrity: sha512-GrSCn37gsIj8fYP3Yn4MLc3sjUN2LWrxD5dOp8G6tfcRhUdazqkZeo5zNAspfyhJdwDMlYk/gXr5xJ0GWxTnuw==} - - '@tangle-network/agent-eval@0.145.1': - resolution: {integrity: sha512-x4ze+gqR0Rr9HfW21ruThxXvKN+BukZ82jkV+XpvWKF3afECL6e3yERisviW/lJtl/uQB7HnLKt0f0YoRzsPng==} + '@tangle-network/agent-eval@0.145.3': + resolution: {integrity: sha512-KPiYjw/SYLFF+mFZF2mjneUmU1Q6SQB6B1PiJRyw3UcnxNxdR8K6M+C7+YGoa/HsEVluhGkD25VPE3T1wTAAww==} engines: {node: '>=20'} hasBin: true '@tangle-network/agent-interface@0.47.0': resolution: {integrity: sha512-sCkr7FOT+SNut8EYtwh/x2JheI+Eer30V5IQVIrO8Fn6csC9pqlnLRuNDJh//pnk4+fsg4yLiiSW8/hL1l9e1w==} - '@tangle-network/agent-knowledge@7.2.3': - resolution: {integrity: sha512-pNt5FJT9RG3GazkBtTbgjHSkndbQDZZ4mmmzr3KRlgvaOucUfmvoN4b1dLmMmrOovK/CSNZYS0SF+X47dbpXzw==} + '@tangle-network/agent-knowledge@7.2.4': + resolution: {integrity: sha512-xzDAH7YuMD3ijO4gnhvtWz92OvrQ5mlvjTco4tea5fBVkQJVZ1YfPVvgnJ0tTmJXSTe9hqpLiKUTZ5jqe9gnxg==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.145.0 <0.146.0' + '@tangle-network/agent-eval': '>=0.145.2 <0.146.0' '@tangle-network/agent-interface': 0.47.0 - '@tangle-network/agent-profile-materialize@0.13.1': - resolution: {integrity: sha512-2g/F8ABiJ6gB8lifTKKIMGPBgYjCyPHQF93HF3g7c5d0D5Rh8cEZ9WGRvggYjOm4HPi3/4PViG6VKEmr5sXwIg==} + '@tangle-network/agent-profile-materialize@0.14.0': + resolution: {integrity: sha512-LfuyhtrPFvjMvtmZoFoPQehk+8I2e3urAlozEzm9MGQAstkPhsv6lJ/M7mpJYSwD6UVm42Q2GhqgRASi0b8lww==} peerDependencies: '@tangle-network/agent-interface': 0.47.0 @@ -539,14 +536,14 @@ packages: peerDependencies: '@tangle-network/sandbox': '>=0.17.0 <1.0.0' - '@tangle-network/agent-runtime@0.132.10': - resolution: {integrity: sha512-Nj3Z+g6LjWo75rfve+RLN5FzNi22ypLPOYAXqGN3sSumJEJd8YOaH8v+AwAGiO8WrWlHkTTdDZXHW387LOQVvg==} + '@tangle-network/agent-runtime@0.132.12': + resolution: {integrity: sha512-ugMP/b5I7+viMSAHS54RyQOTCDPuc23z0Dh3lcjZEIQg42RPZBOji5vHvjClpdNyvBb2i7jREsc9JF7vPBixJA==} engines: {node: '>=22.13.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.145.0 <0.146.0' + '@tangle-network/agent-eval': '>=0.145.2 <0.146.0' '@tangle-network/agent-interface': 0.47.0 - '@tangle-network/sandbox': '>=0.19.4 <0.20.0' + '@tangle-network/sandbox': '>=0.21.1 <0.22.0' peerDependenciesMeta: '@tangle-network/sandbox': optional: true @@ -554,8 +551,8 @@ packages: '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} - '@tangle-network/sandbox@0.19.6': - resolution: {integrity: sha512-pEJEKiZm7ygxp/aXmCrTX/EUta7kwPjf9+a0ENTyr5lp1wRBiOyCs/67cnzS3xCR/w+4J46urkq6LisPmiXtXw==} + '@tangle-network/sandbox@0.21.1': + resolution: {integrity: sha512-xlqI9fxq9TLCOnmcxOU2XgOD5Ls6IOxLAbfa3SxVa/wUNnUCHtnwRHYteF7zUCwngVs6HR01+9dyChc0Q8m2tA==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -1139,21 +1136,16 @@ snapshots: '@noble/hashes@1.8.0': {} - '@tangle-network/agent-core@0.5.4': - dependencies: - '@tangle-network/agent-interface': 0.47.0 - zod: 4.4.3 - - '@tangle-network/agent-core@0.6.0': + '@tangle-network/agent-core@0.6.1': dependencies: '@tangle-network/agent-interface': 0.47.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.145.1': + '@tangle-network/agent-eval@0.145.3': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.1.0(hono@4.13.0) - '@tangle-network/agent-core': 0.5.4 + '@tangle-network/agent-core': 0.6.1 '@tangle-network/agent-interface': 0.47.0 '@tangle-network/agent-trace-contract': 1.0.2 hono: 4.13.0 @@ -1167,14 +1159,14 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@7.2.3(@tangle-network/agent-eval@0.145.1)(@tangle-network/agent-interface@0.47.0)': + '@tangle-network/agent-knowledge@7.2.4(@tangle-network/agent-eval@0.145.3)(@tangle-network/agent-interface@0.47.0)': dependencies: - '@tangle-network/agent-eval': 0.145.1 + '@tangle-network/agent-eval': 0.145.3 '@tangle-network/agent-interface': 0.47.0 proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.13.1(@tangle-network/agent-interface@0.47.0)': + '@tangle-network/agent-profile-materialize@0.14.0(@tangle-network/agent-interface@0.47.0)': dependencies: '@tangle-network/agent-interface': 0.47.0 @@ -1183,22 +1175,22 @@ snapshots: '@tangle-network/agent-interface': 0.47.0 undici: 7.29.0 - '@tangle-network/agent-provider-tangle@0.6.3(@tangle-network/sandbox@0.19.6)': + '@tangle-network/agent-provider-tangle@0.6.3(@tangle-network/sandbox@0.21.1)': dependencies: '@tangle-network/agent-interface': 0.47.0 - '@tangle-network/sandbox': 0.19.6 + '@tangle-network/sandbox': 0.21.1 - '@tangle-network/agent-runtime@0.132.10(@tangle-network/agent-eval@0.145.1)(@tangle-network/agent-interface@0.47.0)(@tangle-network/sandbox@0.19.6)': + '@tangle-network/agent-runtime@0.132.12(@tangle-network/agent-eval@0.145.3)(@tangle-network/agent-interface@0.47.0)(@tangle-network/sandbox@0.21.1)': dependencies: - '@tangle-network/agent-core': 0.5.4 - '@tangle-network/agent-eval': 0.145.1 + '@tangle-network/agent-core': 0.6.1 + '@tangle-network/agent-eval': 0.145.3 '@tangle-network/agent-interface': 0.47.0 - '@tangle-network/agent-knowledge': 7.2.3(@tangle-network/agent-eval@0.145.1)(@tangle-network/agent-interface@0.47.0) - '@tangle-network/agent-profile-materialize': 0.13.1(@tangle-network/agent-interface@0.47.0) + '@tangle-network/agent-knowledge': 7.2.4(@tangle-network/agent-eval@0.145.3)(@tangle-network/agent-interface@0.47.0) + '@tangle-network/agent-profile-materialize': 0.14.0(@tangle-network/agent-interface@0.47.0) '@tangle-network/agent-trace-contract': 1.0.2 tar-stream: 3.2.0 optionalDependencies: - '@tangle-network/sandbox': 0.19.6 + '@tangle-network/sandbox': 0.21.1 transitivePeerDependencies: - bare-abort-controller - bare-buffer @@ -1206,9 +1198,9 @@ snapshots: '@tangle-network/agent-trace-contract@1.0.2': {} - '@tangle-network/sandbox@0.19.6': + '@tangle-network/sandbox@0.21.1': dependencies: - '@tangle-network/agent-core': 0.6.0 + '@tangle-network/agent-core': 0.6.1 '@tangle-network/agent-interface': 0.47.0 zod: 4.4.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2c1dfa6..bcfec34 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -10,14 +10,14 @@ allowBuilds: node-pty: true ignoreScripts: false minimumReleaseAgeExclude: - - '@tangle-network/agent-runtime@0.126.0 || 0.128.0 || 0.131.0 || 0.131.1 || 0.131.2 || 0.131.3 || 0.131.4 || 0.131.5 || 0.131.6 || 0.131.7 || 0.132.0 || 0.132.4 || 0.132.6 || 0.132.9 || 0.132.10' - - '@tangle-network/agent-eval@0.143.0 || 0.144.0 || 0.144.1 || 0.144.3 || 0.144.4 || 0.144.6 || 0.144.8 || 0.144.10 || 0.144.11 || 0.144.13 || 0.145.0 || 0.145.1' + - '@tangle-network/agent-runtime@0.126.0 || 0.128.0 || 0.131.0 || 0.131.1 || 0.131.2 || 0.131.3 || 0.131.4 || 0.131.5 || 0.131.6 || 0.131.7 || 0.132.0 || 0.132.4 || 0.132.6 || 0.132.9 || 0.132.10 || 0.132.11 || 0.132.12' + - '@tangle-network/agent-eval@0.143.0 || 0.144.0 || 0.144.1 || 0.144.3 || 0.144.4 || 0.144.6 || 0.144.8 || 0.144.10 || 0.144.11 || 0.144.13 || 0.145.0 || 0.145.1 || 0.145.2 || 0.145.3' - '@tangle-network/agent-interface@0.46.0 || 0.46.1 || 0.47.0' - - '@tangle-network/agent-knowledge@7.0.8 || 7.1.2 || 7.1.3 || 7.2.0 || 7.2.2 || 7.2.3' - - '@tangle-network/agent-profile-materialize@0.10.2 || 0.13.1' + - '@tangle-network/agent-knowledge@7.0.8 || 7.1.2 || 7.1.3 || 7.2.0 || 7.2.2 || 7.2.3 || 7.2.4' + - '@tangle-network/agent-profile-materialize@0.10.2 || 0.13.1 || 0.14.0' - '@tangle-network/agent-provider-cli-bridge@0.4.2 || 0.4.3 || 0.5.0 || 0.6.0' - '@tangle-network/agent-provider-tangle@0.6.0 || 0.6.1 || 0.6.2 || 0.6.3' - - '@tangle-network/sandbox@0.17.3 || 0.18.0 || 0.19.2 || 0.19.3 || 0.19.4 || 0.19.5 || 0.19.6' + - '@tangle-network/sandbox@0.17.3 || 0.18.0 || 0.19.2 || 0.19.3 || 0.19.4 || 0.19.5 || 0.19.6 || 0.21.0 || 0.21.1' - hono@4.13.0 - '@esbuild/aix-ppc64@0.28.2' - '@esbuild/android-arm64@0.28.2' @@ -46,4 +46,4 @@ minimumReleaseAgeExclude: - '@esbuild/win32-ia32@0.28.2' - '@esbuild/win32-x64@0.28.2' - esbuild@0.28.2 - - '@tangle-network/agent-core@0.5.4 || 0.6.0' + - '@tangle-network/agent-core@0.5.4 || 0.6.0 || 0.6.1' diff --git a/scripts/live-bridge/capture.mjs b/scripts/live-bridge/capture.mjs index 1bda187..49a2151 100644 --- a/scripts/live-bridge/capture.mjs +++ b/scripts/live-bridge/capture.mjs @@ -3,7 +3,7 @@ import { redactString } from './redaction.mjs' const defaultHoldChars = 512 const maxPendingChars = 8_192 const sensitiveBoundaryPattern = - /(?:bearer\s+[A-Za-z0-9._~+/=-]*|(?:token|access[-_]?token|refresh[-_]?token|api[-_]?key|secret|client[-_]?secret|password|authorization|credential)\s*[:=]\s*[^,\s;&#]*|https?:\/\/[^/\s@]*|\b(?:sk|pk|rk)-[A-Za-z0-9_-]*|\bgh[pousr]_[A-Za-z0-9_]*)$/iu + /(?:bearer\s+[A-Za-z0-9._~+/=-]*|(?:token|access[-_ ]*token|refresh[-_ ]*token|api[-_ ]*key|secret|client[-_ ]*secret|password|authorization|credential)\s*[:=]\s*[^,\s;&#]*|https?:\/\/[^/\s@]*|\b(?:sk|pk|rk)-[A-Za-z0-9_-]*|\bgh[pousr]_[A-Za-z0-9_]*)$/iu const sensitiveMarkers = Object.freeze([ 'http://', 'https://', diff --git a/scripts/live-bridge/redaction.mjs b/scripts/live-bridge/redaction.mjs index 8656420..fc22cb7 100644 --- a/scripts/live-bridge/redaction.mjs +++ b/scripts/live-bridge/redaction.mjs @@ -54,7 +54,7 @@ export function redactString(value, secrets = secretValues()) { .replace(/\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}\b/gu, '[redacted]') .replace(/\bgh[pousr]_[A-Za-z0-9_]{20,}\b/gu, '[redacted]') .replace( - /((?:token|access[-_]?token|refresh[-_]?token|api[-_]?key|secret|client[-_]?secret|password|authorization|credential)\s*[:=]\s*)[^,\s;&#]+/giu, + /((?:token|access[-_ ]*token|refresh[-_ ]*token|api[-_ ]*key|secret|client[-_ ]*secret|password|authorization|credential)\s*[:=]\s*)[^,\s;&#]+/giu, '$1[redacted]', ) .replace( diff --git a/scripts/live-demo.test.mjs b/scripts/live-demo.test.mjs index f2d59ec..3d03e18 100644 --- a/scripts/live-demo.test.mjs +++ b/scripts/live-demo.test.mjs @@ -304,6 +304,7 @@ test('public capture rejects the credential patterns mirrored from the sanitizer const filler = 'A'.repeat(32) const cases = [ ['API key assignment', `api${'_key'}=${filler}`], + ['API key phrase', `API key: ${filler}`], ['Bearer value', `Bearer ${'B'.repeat(24)}`], ['OpenAI-style key', `${'s' + 'k-'}${filler}`], ['GitHub classic token', `gh${'p_'}${filler.slice(0, 20)}`], diff --git a/scripts/live-demo/public-safety.mjs b/scripts/live-demo/public-safety.mjs index 3e76cba..3336122 100644 --- a/scripts/live-demo/public-safety.mjs +++ b/scripts/live-demo/public-safety.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' // Keep these patterns aligned with src/domain/secret-sanitizer.ts. const SECRET_ASSIGNMENT = - /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;}\])}]*)/iu + /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_ -]*key|access[_ -]*key|private[_ -]*key|client[_ -]*secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;}\])}]*)/iu const BEARER_ASSIGNMENT = /\bBearer(?:\s+|\s*=\s*)[^\s,;]*/iu const BARE_CREDENTIAL = /(?:sk-[A-Za-z0-9_-]{20,}|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{30,}|xox[baprs]-[A-Za-z0-9-]{20,})/u diff --git a/scripts/live-required.test.mjs b/scripts/live-required.test.mjs index ee2cae4..fe324b4 100644 --- a/scripts/live-required.test.mjs +++ b/scripts/live-required.test.mjs @@ -22,6 +22,7 @@ import { } from './live-required/contracts.mjs' import { closeSession, + configEvidence, initializedSession, prepareProductionWorkspace, runHeadlessTurn, @@ -54,6 +55,25 @@ test('cloud execution stress reports exact small-sample latency distributions', }) }) +test('config evidence accepts connections without optional provider options', () => { + assert.deepEqual( + configEvidence({ + endpoint: { scheme: 'https', host: 'sandbox.tangle.tools' }, + connection: { id: 'connection-test', kind: 'tangle-sandbox' }, + credentialConfigured: true, + profile: { model: { default: 'glm-5.2' }, harness: 'opencode' }, + }), + { + endpoint: { scheme: 'https', host: 'sandbox.tangle.tools' }, + connectionId: 'connection-test', + connectionKind: 'tangle-sandbox', + credentialConfigured: true, + model: 'glm-5.2', + runner: 'opencode', + }, + ) +}) + function protectedEnvironment() { const environment = { ...process.env } for (const name of [ @@ -785,9 +805,11 @@ test('generated live credentials use the same protected store as production Brai } }) -test('generated credential cleanup reports failure and succeeds when retried', async () => { +test('generated credential cleanup disposes failed removal, cleans root, and retains evidence', async () => { let removeAttempts = 0 let disposeCalls = 0 + let rootRemovalAttempts = 0 + const secret = 'live-required-cleanup-retry-canary-822f' const config = await prepareProductionWorkspace({ repository, environment: protectedEnvironment(), @@ -796,7 +818,7 @@ test('generated credential cleanup reports failure and succeeds when retried', a model: 'glm-5.2', runner: 'cli-base', provider: 'tangle', - credentialValue: 'live-required-cleanup-retry-canary-822f', + credentialValue: secret, credentialContextFactory: () => ({ store: { async store(input) { @@ -804,31 +826,123 @@ test('generated credential cleanup reports failure and succeeds when retried', a }, async remove() { removeAttempts += 1 - if (removeAttempts === 1) throw new Error('transient removal failure') + if (removeAttempts === 1) throw new Error(`transient removal failure: ${secret}`) }, }, dispose() { disposeCalls += 1 }, }), + removeTemporaryRoot: async (root) => { + rootRemovalAttempts += 1 + if (rootRemovalAttempts === 1) + throw new Error('transient temporary-root cleanup failure after credential failure') + await rm(root, { recursive: true, force: true }) + }, }) await assert.rejects( () => config.cleanup(), - (error) => error?.code === 'PROTECTED_CREDENTIAL_CLEANUP_FAILED', + (error) => { + assert(error instanceof AggregateError) + assert.equal(error.errors.length, 2) + assert.equal(error.errors[0]?.code, 'PROTECTED_CREDENTIAL_CLEANUP_FAILED') + assert.equal( + error.errors[1]?.message, + 'transient temporary-root cleanup failure after credential failure', + ) + assert.deepEqual(error?.cleanupEvidence, { + credentialRemoved: false, + temporaryRootRemoved: false, + }) + assert.doesNotMatch(safeMessage(error), new RegExp(secret, 'u')) + return true + }, ) await access(config.root) assert.equal(removeAttempts, 1) - assert.equal(disposeCalls, 0) + assert.equal(disposeCalls, 1) + assert.equal(rootRemovalAttempts, 1) - await config.cleanup() + assert.deepEqual(await config.cleanup(), { + credentialRemoved: true, + temporaryRootRemoved: true, + }) await assert.rejects(() => access(config.root), /ENOENT/u) assert.equal(removeAttempts, 2) - assert.equal(disposeCalls, 1) + assert.equal(disposeCalls, 2) + assert.equal(rootRemovalAttempts, 2) - await config.cleanup() + assert.deepEqual(await config.cleanup(), { + credentialRemoved: true, + temporaryRootRemoved: true, + }) assert.equal(removeAttempts, 2) - assert.equal(disposeCalls, 1) + assert.equal(disposeCalls, 2) + assert.equal(rootRemovalAttempts, 2) +}) + +test('temporary-root cleanup retries after a transient failure without repeating credential removal', async () => { + let credentialRemovalAttempts = 0 + let disposeCalls = 0 + let rootRemovalAttempts = 0 + const config = await prepareProductionWorkspace({ + repository, + environment: protectedEnvironment(), + kind: 'temporary-root-retry', + endpoint: 'https://router.tangle.tools', + model: 'glm-5.2', + runner: 'cli-base', + provider: 'tangle', + credentialValue: 'live-required-root-cleanup-canary-154d', + credentialContextFactory: () => ({ + store: { + async store(input) { + return input.ref + }, + async remove() { + credentialRemovalAttempts += 1 + }, + }, + dispose() { + disposeCalls += 1 + }, + }), + removeTemporaryRoot: async (root) => { + rootRemovalAttempts += 1 + if (rootRemovalAttempts === 1) throw new Error('transient temporary-root cleanup failure') + await rm(root, { recursive: true, force: true }) + }, + }) + + try { + await assert.rejects( + () => config.cleanup(), + (error) => { + assert.equal(error?.message, 'transient temporary-root cleanup failure') + assert.deepEqual(error?.cleanupEvidence, { + credentialRemoved: true, + temporaryRootRemoved: false, + }) + return true + }, + ) + await access(config.root) + assert.equal(credentialRemovalAttempts, 1) + assert.equal(disposeCalls, 1) + assert.equal(rootRemovalAttempts, 1) + + assert.deepEqual(await config.cleanup(), { + credentialRemoved: true, + temporaryRootRemoved: true, + }) + await assert.rejects(() => access(config.root), /ENOENT/u) + assert.equal(credentialRemovalAttempts, 1) + assert.equal(disposeCalls, 1) + assert.equal(rootRemovalAttempts, 2) + } finally { + await config.cleanup().catch(() => undefined) + } }) test('configured real-path assertion failures emit failed release evidence', async () => { diff --git a/scripts/live-required/headless.mjs b/scripts/live-required/headless.mjs index ccb5469..61a6b3f 100644 --- a/scripts/live-required/headless.mjs +++ b/scripts/live-required/headless.mjs @@ -1,7 +1,7 @@ import { randomBytes, randomUUID } from 'node:crypto' import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { join, resolve } from 'node:path' import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { RpcSession } from '../live-bridge/process.mjs' @@ -62,6 +62,30 @@ function liveDataDirectory(root, platform = process.platform) { return join(root, '.xdg-data', 'braid') } +function combineFailures(failures, message) { + const present = failures.filter((failure) => failure !== undefined) + if (present.length === 0) return undefined + if (present.length === 1) return present[0] + return new AggregateError(present, message) +} + +function cleanupEvidence(credentialRemoved, temporaryRootRemoved) { + return Object.freeze({ credentialRemoved, temporaryRootRemoved }) +} + +function withCleanupEvidence(error, evidence) { + if (error !== null && (typeof error === 'object' || typeof error === 'function')) { + try { + Object.defineProperty(error, 'cleanupEvidence', { + configurable: true, + enumerable: false, + value: evidence, + }) + } catch {} + } + return error +} + async function installGeneratedCredential({ repository, workspace, @@ -74,10 +98,17 @@ async function installGeneratedCredential({ createCredentialContext, }) { let context + let contextFactory const portRef = `cred:v1:${credentialId}` const secret = Buffer.from(value) + const contextOptions = { + workspace, + configPath, + databaseKeyFile, + dataDirectory, + } try { - const contextFactory = + contextFactory = createCredentialContext ?? ( await import( @@ -86,42 +117,91 @@ async function installGeneratedCredential({ ).href ) ).createProductionCredentialContext - context = contextFactory({ - workspace, - configPath, - databaseKeyFile, - dataDirectory, - }) - if (context === undefined) throw new Error('The protected credential context was not created') + + const createContext = () => { + const candidate = contextFactory(contextOptions) + if (candidate === undefined) + throw new Error('The protected credential context was not created') + return candidate + } + + context = createContext() await context.store.store({ ref: portRef, value: secret, label: `Braid live ${kind} release check`, }) let removed = false + let removalPromise return { credentialId, remove: async () => { - if (removed) return - try { - await context.store.remove(portRef) - context.dispose() - removed = true - } catch (error) { - throw protectedUnavailable( - 'PROTECTED_CREDENTIAL_CLEANUP_FAILED', - `The temporary ${kind} credential could not be removed`, - error, + if (removed) return { credentialRemoved: true } + if (removalPromise !== undefined) return removalPromise + removalPromise = (async () => { + let current + let removalError + let disposeError + try { + current = context ?? createContext() + context = undefined + await current.store.remove(portRef) + removed = true + } catch (error) { + removalError = protectedUnavailable( + 'PROTECTED_CREDENTIAL_CLEANUP_FAILED', + `The temporary ${kind} credential could not be removed`, + error, + ) + } finally { + try { + current?.dispose() + } catch (error) { + disposeError = protectedUnavailable( + 'PROTECTED_CREDENTIAL_CLEANUP_FAILED', + `The temporary ${kind} credential context could not be disposed`, + error, + ) + } + } + const failure = combineFailures( + [removalError, disposeError], + `The temporary ${kind} credential cleanup failed`, ) + if (failure !== undefined) { + if (removalError !== undefined && disposeError !== undefined) { + throw protectedUnavailable( + 'PROTECTED_CREDENTIAL_CLEANUP_FAILED', + `The temporary ${kind} credential cleanup failed`, + failure, + ) + } + throw failure + } + return { credentialRemoved: true } + })() + try { + return await removalPromise + } finally { + removalPromise = undefined } }, } } catch (error) { - context?.dispose() + let disposeError + try { + context?.dispose() + } catch (cleanupError) { + disposeError = cleanupError + } + const failure = combineFailures( + [error, disposeError], + `The supplied ${kind} credential could not be installed in Braid's protected workspace credential store`, + ) throw protectedUnavailable( 'PROTECTED_CREDENTIAL_STORE_UNAVAILABLE', `The supplied ${kind} credential could not be installed in Braid's protected workspace credential store`, - error, + failure, ) } finally { secret.fill(0) @@ -168,41 +248,43 @@ export async function prepareProductionWorkspace({ runner, provider, connectionName, + providerOptions = {}, credentialRef, credentialValue, credentialContextFactory, + removeTemporaryRoot = (path) => rm(path, { recursive: true, force: true }), }) { const root = await mkdtemp(join(tmpdir(), 'braid-live-required-')) - const kindId = kind.replace(/[^A-Za-z0-9._~-]/gu, '-') - const workspace = join(root, 'workspace') - const configDirectory = join(workspace, '.braid') - const profileDirectory = join(configDirectory, 'profiles') - const databaseKeyFile = join(root, 'database.key') - const configPath = join(configDirectory, 'config.json') - const dataDirectory = liveDataDirectory(root) - const generatedCredentialId = - credentialValue === undefined - ? undefined - : `credential-live-${kindId}-${randomUUID().replaceAll('-', '')}` - const selectedCredentialId = generatedCredentialId ?? credentialRef - if (selectedCredentialId !== undefined) validCredentialId(selectedCredentialId) - const profile = profileFor({ kind, model, runner, provider }) - const profileFile = `profile-${kindId}.json` - const profilePath = join(profileDirectory, profileFile) - const now = timestamp() - const connection = { - id: `connection-live-${kindId}`, - kind, - name: connectionName ?? `Live ${kind}`, - endpoint, - ...(selectedCredentialId === undefined ? {} : { credentialRef: selectedCredentialId }), - providerOptions: { transport: 'https' }, - createdAt: now, - updatedAt: now, - lastHealth: { status: 'unknown' }, - } let generatedCredential try { + const kindId = kind.replace(/[^A-Za-z0-9._~-]/gu, '-') + const workspace = join(root, 'workspace') + const configDirectory = join(workspace, '.braid') + const profileDirectory = join(configDirectory, 'profiles') + const databaseKeyFile = join(root, 'database.key') + const configPath = join(configDirectory, 'config.json') + const dataDirectory = liveDataDirectory(root) + const generatedCredentialId = + credentialValue === undefined + ? undefined + : `credential-live-${kindId}-${randomUUID().replaceAll('-', '')}` + const selectedCredentialId = generatedCredentialId ?? credentialRef + if (selectedCredentialId !== undefined) validCredentialId(selectedCredentialId) + const profile = profileFor({ kind, model, runner, provider }) + const profileFile = `profile-${kindId}.json` + const profilePath = join(profileDirectory, profileFile) + const now = timestamp() + const connection = { + id: `connection-live-${kindId}`, + kind, + name: connectionName ?? `Live ${kind}`, + endpoint, + ...(selectedCredentialId === undefined ? {} : { credentialRef: selectedCredentialId }), + providerOptions: { ...providerOptions, transport: 'https' }, + createdAt: now, + updatedAt: now, + lastHealth: { status: 'unknown' }, + } await mkdir(profileDirectory, { recursive: true, mode: 0o700 }) await writeFile(databaseKeyFile, `${randomBytes(32).toString('hex')}\n`, { mode: 0o600 }) await writeFile(profilePath, `${JSON.stringify(profile, null, 2)}\n`, { mode: 0o600 }) @@ -235,7 +317,9 @@ export async function prepareProductionWorkspace({ kind, createCredentialContext: credentialContextFactory, }) - let cleaned = false + let temporaryRootRemoved = false + let cleanupPromise + let lastCleanupEvidence = cleanupEvidence(generatedCredential === undefined, false) return { root, workspace, @@ -249,23 +333,63 @@ export async function prepareProductionWorkspace({ credentialConfigured: selectedCredentialId !== undefined, environment: childEnvironment(environment, root, join(root, 'state.sqlite')), cleanup: async () => { - if (cleaned) return - await generatedCredential?.remove() - await rm(root, { recursive: true, force: true }) - cleaned = true + if (temporaryRootRemoved) return lastCleanupEvidence + if (cleanupPromise !== undefined) return cleanupPromise + cleanupPromise = (async () => { + let credentialResult = { + credentialRemoved: generatedCredential === undefined, + } + let credentialError + let workspaceError + try { + if (generatedCredential !== undefined) + credentialResult = await generatedCredential.remove() + } catch (error) { + credentialError = error + } finally { + try { + await removeTemporaryRoot(root) + temporaryRootRemoved = true + } catch (error) { + workspaceError = error + } + } + lastCleanupEvidence = cleanupEvidence( + credentialResult.credentialRemoved === true, + temporaryRootRemoved, + ) + const failure = combineFailures( + [credentialError, workspaceError], + 'The live workspace cleanup failed', + ) + if (failure !== undefined) throw withCleanupEvidence(failure, lastCleanupEvidence) + return lastCleanupEvidence + })() + try { + return await cleanupPromise + } finally { + cleanupPromise = undefined + } }, } } catch (error) { + const failures = [error] try { await generatedCredential?.remove() - await rm(root, { recursive: true, force: true }) } catch (cleanupError) { - throw new AggregateError( - [error, cleanupError], - 'The live workspace failed during setup and could not be cleaned up', - ) + failures.push(cleanupError) } - throw error + try { + await removeTemporaryRoot(root) + } catch (cleanupError) { + failures.push(cleanupError) + } + throw ( + combineFailures( + failures, + 'The live workspace failed during setup and could not be cleaned up', + ) ?? error + ) } } @@ -430,6 +554,7 @@ export async function runHeadlessCancellation({ binary, config, marker, prompt, } export function configEvidence(config) { + const providerOptions = config.connection.providerOptions ?? {} return { endpoint: config.endpoint, connectionId: config.connection.id, @@ -437,6 +562,10 @@ export function configEvidence(config) { credentialConfigured: config.credentialConfigured, model: config.profile.model.default, runner: config.profile.harness, + ...(providerOptions.lifecycle === undefined ? {} : { lifecycle: providerOptions.lifecycle }), + ...(providerOptions.idleTtlSeconds === undefined + ? {} + : { idleTtlSeconds: providerOptions.idleTtlSeconds }), } } diff --git a/scripts/live-required/tangle-sandbox-braid-soak.mjs b/scripts/live-required/tangle-sandbox-braid-soak.mjs index 7e54d92..61bcd47 100644 --- a/scripts/live-required/tangle-sandbox-braid-soak.mjs +++ b/scripts/live-required/tangle-sandbox-braid-soak.mjs @@ -10,6 +10,596 @@ const DEFAULT_RUNS = 3 const DEFAULT_CONCURRENCY = 2 const MAX_RUNS = 20 const MAX_CONCURRENCY = 4 +const ACCOUNT_USAGE_FIELDS = [ + 'activeSandboxes', + 'totalSandboxes', + 'computeMinutes', + 'gpuSeconds', + 'gpuCostUsd', +] +const TELEMETRY_FIELDS = [ + 'tokens', + 'cost', + 'endToEndDuration', + 'model', + 'environment', + 'runtimeEndpoint', + 'machine', + 'region', + 'requestedResources', + 'resourceSample', + 'account', +] +const CONTROL_REF_FIELDS = [ + 'provider', + 'environmentId', + 'sessionId', + 'executionId', + 'runId', + 'requestDigest', +] +const RUN_SNAPSHOT_NAMES = ['first', 'resumed', 'followUp', 'cancelled'] + +function record(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : undefined +} + +function nonEmptyString(value) { + return typeof value === 'string' && value.length > 0 +} + +function finiteNonNegative(value) { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 +} + +function exactControlRef(value) { + return ( + record(value) !== undefined && + value.provider === 'tangle-sandbox' && + CONTROL_REF_FIELDS.every((field) => nonEmptyString(value[field])) && + /^sha256:[0-9a-f]{64}$/u.test(value.requestDigest) + ) +} + +function sameControlRef(left, right) { + return CONTROL_REF_FIELDS.every((field) => left?.[field] === right?.[field]) +} + +function usageEntry(proof, phase) { + return Array.isArray(proof?.usage) + ? proof.usage.find((entry) => entry?.phase === phase) + : undefined +} + +function measuredUsageDelta(proof) { + const before = usageEntry(proof, 'before') + const after = usageEntry(proof, 'after') + return { + before, + after, + delta: resourceDelta(after?.value, before?.value), + } +} + +function declaredDeltaExpectation(proof, field) { + const sources = [ + proof?.cleanup?.usageDeltaExpectations, + proof?.usageDeltaExpectations, + proof?.accountDeltaExpectations, + ] + for (const source of sources) { + const declaration = source?.[field] + const expectedMax = + declaration?.expectedMax ?? declaration?.max ?? declaration?.expected ?? undefined + if ( + record(declaration) !== undefined && + nonEmptyString(declaration.classification) && + finiteNonNegative(expectedMax) + ) { + return { classification: declaration.classification, expectedMax } + } + } + return undefined +} + +function cumulativeExpectation(proof, field, maxConcurrentRuns, aggregate = false) { + const measuredMs = proof?.timing?.totalMs + const durationMinutes = + typeof measuredMs === 'number' && Number.isFinite(measuredMs) && measuredMs > 0 + ? Math.max(1, Math.ceil(measuredMs / 60_000)) + : undefined + const durationSeconds = + typeof measuredMs === 'number' && Number.isFinite(measuredMs) && measuredMs > 0 + ? Math.max(1, Math.ceil(measuredMs / 1_000)) + : undefined + const concurrencyFactor = aggregate ? 1 : Math.max(1, maxConcurrentRuns) + const declared = declaredDeltaExpectation(proof, field) + + if (field === 'totalSandboxes') { + const matchedCount = proof?.cleanup?.identity?.matchedCount + if (!Number.isSafeInteger(matchedCount) || matchedCount < 0) return undefined + return { + classification: 'retained-resource-creation', + expectedMax: matchedCount * concurrencyFactor, + } + } + if (field === 'computeMinutes') { + if (durationMinutes === undefined) return undefined + return { + classification: 'measured-proof-work', + expectedMax: durationMinutes * concurrencyFactor, + } + } + if (field === 'gpuSeconds') { + if (declared === undefined) return { classification: 'no-measured-gpu-work', expectedMax: 0 } + if (durationSeconds === undefined) return undefined + return { + classification: declared.classification, + expectedMax: Math.min(declared.expectedMax, durationSeconds * concurrencyFactor), + } + } + if (field === 'gpuCostUsd') { + if (declared === undefined) return { classification: 'no-measured-gpu-work', expectedMax: 0 } + return { + classification: declared.classification, + expectedMax: declared.expectedMax * concurrencyFactor, + } + } + return undefined +} + +function addUsageFailures(failures, proof, label, maxConcurrentRuns, aggregate = false) { + const { before, after, delta } = measuredUsageDelta(proof) + if (before?.status !== 'observed' || record(before?.value) === undefined) { + failures.push(`${label} usage-before observation was missing`) + } + if (after?.status !== 'observed' || record(after?.value) === undefined) { + failures.push(`${label} usage-after observation was missing`) + } + for (const entry of [before, after]) { + for (const field of ACCOUNT_USAGE_FIELDS) { + if (!finiteNonNegative(entry?.value?.[field])) { + failures.push(`${label} usage ${entry?.phase ?? 'unknown'} ${field} was unknown`) + } + } + } + + if (!Array.isArray(delta?.unknownFields) || delta.unknownFields.length > 0) { + failures.push( + `${label} account usage delta had unknown fields: ${delta?.unknownFields?.join(', ') ?? 'unavailable'}`, + ) + } + if (delta?.activeSandboxes !== 0) { + failures.push( + `${label} account active-resource delta was ${delta?.activeSandboxes ?? 'unknown'}, expected 0`, + ) + } + if (!aggregate && proof?.cleanup?.activeResourceDelta !== 0) { + failures.push( + `${label} cleanup activeResourceDelta was ${proof?.cleanup?.activeResourceDelta ?? 'unknown'}, expected 0`, + ) + } + + const reported = proof?.cleanup?.usageDelta + if (record(reported) === undefined) { + failures.push(`${label} cleanup usage delta was missing`) + } else { + for (const field of ACCOUNT_USAGE_FIELDS) { + if (reported[field] !== delta?.[field]) { + failures.push( + `${label} cleanup usage delta ${field} did not match before/after observations`, + ) + } + } + if ( + !Array.isArray(reported.unknownFields) || + JSON.stringify(reported.unknownFields) !== JSON.stringify(delta?.unknownFields ?? []) + ) { + failures.push(`${label} cleanup usage delta unknown-field disclosure was invalid`) + } + } + + for (const field of ACCOUNT_USAGE_FIELDS.slice(1)) { + const value = delta?.[field] + if (!finiteNonNegative(value)) { + failures.push(`${label} account ${field} delta was unknown`) + continue + } + const expectation = cumulativeExpectation(proof, field, maxConcurrentRuns, aggregate) + if (expectation === undefined) { + failures.push(`${label} account ${field} delta had no measured-work bound`) + continue + } + if (value > expectation.expectedMax) { + failures.push( + `${label} account ${field} delta ${value} exceeded ${expectation.classification} bound ${expectation.expectedMax}`, + ) + } + if (value > 0 && !nonEmptyString(expectation.classification)) { + failures.push(`${label} account ${field} delta lacked an explicit classification`) + } + if (field === 'gpuCostUsd' && value > 0 && delta?.gpuSeconds <= 0) { + failures.push(`${label} account gpuCostUsd delta was not tied to measured GPU work`) + } + } + return { before, after, delta } +} + +function addRunSnapshotFailures(failures, runs, name) { + const snapshot = runs?.[name] + if (record(snapshot) === undefined) { + failures.push(`runs.${name} durable snapshot was missing`) + return undefined + } + for (const field of ['id', 'operationId', 'environmentId', 'providerSessionId', 'status']) { + if (!nonEmptyString(snapshot[field])) failures.push(`runs.${name}.${field} was missing`) + } + if (!exactControlRef(snapshot.controlRef)) { + failures.push(`runs.${name}.controlRef was incomplete`) + } + const observations = snapshot.observations + if (record(observations) === undefined) { + failures.push(`runs.${name}.observations were missing`) + } else { + if (observations.localEnvironmentId !== snapshot.environmentId) { + failures.push(`runs.${name}.observations.localEnvironmentId was not durable`) + } + if (observations.providerEnvironmentId !== snapshot.controlRef?.environmentId) { + failures.push(`runs.${name}.observations.providerEnvironmentId was not durable`) + } + for (const field of ['environmentRecord', 'run', 'environment']) { + if (record(observations[field]) === undefined) { + failures.push(`runs.${name}.observations.${field} was missing`) + } + } + } + if (!nonEmptyString(snapshot.cursor)) failures.push(`runs.${name}.cursor was missing`) + return snapshot +} + +function proofFailures(proof, { maxConcurrentRuns = 1 } = {}) { + if (proof?.status !== 'passed') return ['status was not passed'] + const failures = [] + const progress = proof.progress + const firstControlRef = progress?.firstControlRef + const freshControlRef = progress?.freshControlRef + if (record(progress) === undefined) { + failures.push('progress identity was missing') + } else { + for (const field of [ + 'firstRunId', + 'cancelRunId', + 'providerEnvironmentId', + 'resumeFromCursor', + 'finalCursor', + ]) { + if (!nonEmptyString(progress[field])) failures.push(`progress.${field} was missing`) + } + if (!exactControlRef(firstControlRef)) failures.push('progress.firstControlRef was incomplete') + if (!exactControlRef(freshControlRef)) failures.push('progress.freshControlRef was incomplete') + if (exactControlRef(firstControlRef) && exactControlRef(freshControlRef)) { + if (!sameControlRef(firstControlRef, freshControlRef)) { + failures.push('restart changed the durable provider control identity') + } + if (progress.providerEnvironmentId !== firstControlRef.environmentId) { + failures.push('progress.providerEnvironmentId did not match firstControlRef') + } + } + } + + const runs = proof.runs + if (record(runs) === undefined) { + failures.push('durable run snapshots were missing') + } else { + const snapshots = Object.fromEntries( + RUN_SNAPSHOT_NAMES.map((name) => [name, addRunSnapshotFailures(failures, runs, name)]), + ) + const first = snapshots.first + const resumed = snapshots.resumed + const followUp = snapshots.followUp + const cancelled = snapshots.cancelled + if (first && progress?.firstRunId !== first.id) + failures.push('first run identity was not durable') + if (first && resumed && first.id !== resumed.id) + failures.push('restart changed the local run identity') + if (cancelled && progress?.cancelRunId !== cancelled.id) { + failures.push('cancellation run identity was not durable') + } + if ( + first && + resumed && + exactControlRef(first.controlRef) && + exactControlRef(resumed.controlRef) + ) { + if (!sameControlRef(first.controlRef, resumed.controlRef)) { + failures.push('restart changed the run control identity') + } + } + if (exactControlRef(firstControlRef)) { + for (const [name, snapshot] of [ + ['first', first], + ['resumed', resumed], + ]) { + if ( + snapshot && + exactControlRef(snapshot.controlRef) && + !sameControlRef(snapshot.controlRef, firstControlRef) + ) { + failures.push(`${name} control identity did not match progress.firstControlRef`) + } + } + } + if ( + exactControlRef(freshControlRef) && + resumed && + exactControlRef(resumed.controlRef) && + !sameControlRef(resumed.controlRef, freshControlRef) + ) { + failures.push('progress.freshControlRef did not match the resumed run') + } + if (first && followUp && cancelled) { + for (const [name, snapshot] of [ + ['followUp', followUp], + ['cancelled', cancelled], + ]) { + if (!exactControlRef(snapshot.controlRef)) continue + if (snapshot.controlRef.environmentId !== first.controlRef?.environmentId) { + failures.push(`${name} changed the retained environment identity`) + } + if (snapshot.controlRef.sessionId !== first.controlRef?.sessionId) { + failures.push(`${name} changed the retained session identity`) + } + if (snapshot.controlRef.executionId === first.controlRef?.executionId) { + failures.push(`${name} reused the first execution identity`) + } + if (snapshot.controlRef.runId === first.controlRef?.runId) { + failures.push(`${name} reused the first provider run identity`) + } + } + } + if (resumed?.status !== 'completed') failures.push('resumed run was not completed') + if (followUp?.status !== 'completed') failures.push('follow-up run was not completed') + if (!['cancelled', 'aborted'].includes(cancelled?.status)) { + failures.push('cancelled run did not reach a cancellation terminal state') + } + } + + const processes = proof.processes + if (record(processes) === undefined) { + failures.push('restart process proof was missing') + } else { + if (processes.first?.signal !== 'SIGKILL' || processes.first?.sent !== true) { + failures.push('restart did not prove active-process disconnect') + } + if (processes.first?.code !== null) failures.push('restart exit signal was not durable') + for (const field of ['first', 'cancelled', 'retry']) { + const cleanup = processes[field]?.cleanup + if ( + cleanup?.exited !== true || + cleanup?.descendantsVerified !== true || + record(cleanup?.exit) === undefined + ) { + failures.push(`processes.${field}.cleanup was incomplete`) + } + } + if (processes.localRunCountAfterReconnect !== 1) { + failures.push('restart did not preserve one local run') + } + if (!/^[0-9a-f]{64}$/u.test(processes.binarySha256 ?? '')) { + failures.push('Braid binary identity was missing') + } + } + + const replay = proof.replay + if (record(replay) === undefined) { + failures.push('restart/replay proof was missing') + } else { + for (const field of ['firstVisibleEventCount', 'freshVisibleEventCount']) { + if (!Number.isSafeInteger(replay[field]) || replay[field] < 1) { + failures.push(`replay.${field} was not non-vacuous`) + } + } + for (const field of [ + 'resumeFromCursor', + 'finalCursor', + 'acknowledgedBeforeKillEventIds', + 'freshVisibleEventIds', + 'acknowledgedAndFreshIntersection', + 'progress', + 'reconnectRequest', + ]) { + if (replay[field] === undefined || replay[field] === null) { + failures.push(`replay.${field} was missing`) + } + } + for (const field of ['resumeFromCursor', 'finalCursor']) { + if (!nonEmptyString(replay[field])) failures.push(`replay.${field} was not durable`) + } + for (const field of ['acknowledgedBeforeKillEventIds', 'freshVisibleEventIds']) { + if ( + !Array.isArray(replay[field]) || + replay[field].length < 1 || + replay[field].some((value) => !nonEmptyString(value)) + ) { + failures.push(`replay.${field} was not a durable event set`) + } + } + if (replay.freshVisibleEventIdsUnique !== true) { + failures.push('replay fresh event uniqueness was missing') + } + if (!Array.isArray(replay.acknowledgedAndFreshIntersection)) { + failures.push('replay exclusive resume proof was missing') + } else if (replay.acknowledgedAndFreshIntersection.length !== 0) { + failures.push('replay reused an acknowledged provider event') + } + if ( + !Number.isSafeInteger(replay.progress?.acknowledgedSequence) || + !Number.isSafeInteger(replay.progress?.firstFreshSequence) || + replay.progress.firstFreshSequence <= replay.progress.acknowledgedSequence + ) { + failures.push('replay cursor progress was missing') + } + if ( + replay.reconnectRequest?.command !== 'reconnect' || + replay.reconnectRequest?.params?.runId !== progress?.firstRunId + ) { + failures.push('replay reconnect request was not durable') + } + } + + const cancellation = proof.cancellation + if (record(cancellation) === undefined) { + failures.push('cancellation proof was missing') + } else { + if (cancellation.first?.type !== 'ack' || !nonEmptyString(cancellation.first.runId)) { + failures.push('first cancellation acknowledgement was missing') + } + if ( + cancellation.sameBody?.type !== 'ack' || + (!cancellation.sameBody.replayed && cancellation.sameBody.outcome !== 'already-applied') + ) { + failures.push('same-body cancellation replay proof was missing') + } + if ( + cancellation.changedBody?.type !== 'error' || + cancellation.changedBody.code !== 'OPERATION_CONFLICT' + ) { + failures.push('changed-body cancellation conflict proof was missing') + } + if ( + cancellation.remote?.settledStatus !== 'cancelled' || + cancellation.remote?.lateResult !== false || + !exactControlRef(cancellation.remote?.controlRef) || + !Array.isArray(cancellation.remote?.samples) || + cancellation.remote.samples.length < 1 + ) { + failures.push('remote cancellation proof was incomplete') + } else if ( + record(runs?.cancelled) !== undefined && + exactControlRef(runs.cancelled.controlRef) && + !sameControlRef(cancellation.remote.controlRef, runs.cancelled.controlRef) + ) { + failures.push('remote cancellation identity did not match the cancelled run') + } + } + + const telemetry = proof.telemetry + const validateTelemetryDisclosure = (disclosure, label) => { + if (record(disclosure) === undefined || disclosure.completeDisclosure !== true) { + failures.push(`${label} complete disclosure was missing`) + return + } + if (!Array.isArray(disclosure.unavailable) || record(disclosure.fields) === undefined) { + failures.push(`${label} availability disclosure was missing`) + return + } + for (const field of TELEMETRY_FIELDS) { + const value = disclosure.fields[field] + if ( + record(value) === undefined || + !nonEmptyString(value.status) || + value.status === 'missing' + ) { + failures.push(`${label}.fields.${field} was missing or silently unavailable`) + } + } + } + validateTelemetryDisclosure(telemetry, 'telemetry') + if (record(telemetry?.runs) === undefined) { + failures.push('telemetry disclosure for every retained run was missing') + } else { + for (const name of RUN_SNAPSHOT_NAMES) { + validateTelemetryDisclosure(telemetry.runs[name], `telemetry.runs.${name}`) + } + } + + const workspace = proof.workspaceVerification + const continuity = proof.followUpEvidence?.continuity + if ( + workspace?.readMatched !== true || + workspace?.continuity?.matched !== true || + workspace?.git?.exitCode !== 0 || + continuity?.matched !== true || + !Number.isSafeInteger(proof.followUpEvidence?.visibleProviderEvents) || + proof.followUpEvidence.visibleProviderEvents < 1 + ) { + failures.push('retained workspace continuity proof was incomplete') + } + + const resourceIdentity = proof.resourceIdentity + if ( + resourceIdentity?.observed !== true || + !nonEmptyString(resourceIdentity.id) || + !nonEmptyString(resourceIdentity.name) || + resourceIdentity.metadata?.owner !== 'braid' || + resourceIdentity.metadata?.lifecycle !== 'retained' || + resourceIdentity.metadata?.providerSessionId !== firstControlRef?.sessionId || + resourceIdentity.id !== firstControlRef?.environmentId + ) { + failures.push('retained resource identity was incomplete') + } + + const cleanup = proof.cleanup + if (record(cleanup) === undefined) { + failures.push('cleanup proof was missing') + } else { + if (cleanup.exactResource !== true || cleanup.mode !== 'exact-owned-resource-set') { + failures.push('exact retained cleanup was not proven') + } + if (cleanup.activeResourceDelta !== 0) { + failures.push( + `cleanup activeResourceDelta was ${cleanup.activeResourceDelta ?? 'unknown'}, expected 0`, + ) + } + if (typeof cleanup.activeResourceDeltaRequired !== 'boolean') { + failures.push('cleanup active-resource requirement was not durable') + } + if (cleanup.usageObservationComplete !== true) { + failures.push('cleanup usage observation was incomplete') + } + const identity = cleanup.identity + if ( + identity?.confirmed !== true || + !Number.isSafeInteger(identity.matchedCount) || + identity.matchedCount !== 1 || + !Array.isArray(identity.removedIds) || + identity.removedIds.length !== 1 || + identity.removedIds[0] !== resourceIdentity?.id || + !Array.isArray(identity.remainingIds) || + identity.remainingIds.length !== 0 || + !Array.isArray(identity.deletions) || + identity.deletions.length < 1 || + identity.deletions.some((deletion) => deletion?.confirmed !== true) + ) { + failures.push('durable exact cleanup identity was incomplete') + } + } + + const accountIdentity = proof.accountIdentityConsistency + if (accountIdentity?.stable !== true || !nonEmptyString(accountIdentity.identityDigest)) { + failures.push('stable account identity was missing') + } + const identities = proof.accountIdentities + if (!Array.isArray(identities)) { + failures.push('durable account identity observations were missing') + } else { + for (const phase of ['before', 'after']) { + const entry = identities.find((candidate) => candidate?.phase === phase) + if (entry?.status !== 'observed' || !nonEmptyString(entry.value?.identityDigest)) { + failures.push(`account identity ${phase} observation was incomplete`) + } + } + } + if ( + record(proof.account) === undefined || + proof.account.identityDigest !== accountIdentity?.identityDigest + ) { + failures.push('execution account identity was not tied to the stable account') + } + + addUsageFailures(failures, proof, 'proof', maxConcurrentRuns) + return [...new Set(failures)] +} function argument(name, argv = process.argv) { const index = argv.indexOf(`--${name}`) @@ -41,15 +631,8 @@ function distribution(values) { } } -function proofPassed(proof) { - return ( - proof?.status === 'passed' && - proof.cleanup?.exactResource === true && - proof.cleanup?.identity?.confirmed === true && - Array.isArray(proof.cleanup.identity.remainingIds) && - proof.cleanup.identity.remainingIds.length === 0 && - proof.accountIdentityConsistency?.stable === true - ) +function proofPassed(proof, maxConcurrentRuns = 1) { + return proofFailures(proof, { maxConcurrentRuns }).length === 0 } function proofIdentity(proof) { @@ -58,7 +641,7 @@ function proofIdentity(proof) { function accountKey(proof) { const account = proof?.accountIdentityConsistency - return account?.stable === true ? `${account.customerId}:${account.billingOwnerId}` : undefined + return account?.stable === true ? account.identityDigest : undefined } function latencySummary(attempts) { @@ -109,28 +692,66 @@ function sessionSpend(attempts) { } function cohortUsage(attempts) { - const before = attempts[0]?.proof?.usage?.find((entry) => entry.phase === 'before')?.value + const beforeEntry = attempts[0]?.proof?.usage?.find((entry) => entry.phase === 'before') const finalAttempt = attempts .toSorted((left, right) => left.completionSequence - right.completionSequence) .at(-1) - const after = finalAttempt?.proof?.usage?.find((entry) => entry.phase === 'after')?.value + const afterEntry = finalAttempt?.proof?.usage?.find((entry) => entry.phase === 'after') + const before = beforeEntry?.value + const after = afterEntry?.value const delta = resourceDelta(after, before) return { - complete: before !== undefined && before !== null && after !== undefined && after !== null, + complete: + beforeEntry?.status === 'observed' && + afterEntry?.status === 'observed' && + record(before) !== undefined && + record(after) !== undefined && + delta.unknownFields.length === 0, before: before ?? null, after: after ?? null, delta, } } -function cohortFailures(attempts, requestedRuns, usage) { +function cohortAccountFailures(attempts, usage) { + const failures = [] + if (!usage.complete) { + failures.push('cohort account usage delta was unavailable or unknown') + return failures + } + if (usage.delta.activeSandboxes !== 0) { + failures.push(`cohort active-resource delta was ${usage.delta.activeSandboxes}, expected 0`) + } + for (const field of ACCOUNT_USAGE_FIELDS.slice(1)) { + const value = usage.delta[field] + if (!finiteNonNegative(value)) { + failures.push(`cohort account ${field} delta was unknown`) + continue + } + const expectedMax = attempts.reduce((total, attempt) => { + const expectation = cumulativeExpectation(attempt.proof, field, 1, true) + return total + (expectation?.expectedMax ?? 0) + }, 0) + if (value > expectedMax) { + failures.push( + `cohort account ${field} delta ${value} exceeded measured-work bound ${expectedMax}`, + ) + } + } + return failures +} + +function cohortFailures(attempts, requestedRuns, concurrency, usage) { const failures = [] if (attempts.length !== requestedRuns) { failures.push(`attempted ${attempts.length} of ${requestedRuns} requested runs`) } for (const attempt of attempts) { - if (!proofPassed(attempt.proof)) + const proofErrors = proofFailures(attempt.proof, { maxConcurrentRuns: concurrency }) + if (proofErrors.length > 0) { failures.push(`run ${attempt.index + 1} did not pass exact proof`) + failures.push(...proofErrors.map((error) => `run ${attempt.index + 1}: ${error}`)) + } } const uniqueField = (label, values) => { @@ -162,20 +783,20 @@ function cohortFailures(attempts, requestedRuns, usage) { } else if (new Set(accounts).size !== 1) { failures.push('Sandbox account identity changed during the cohort') } - if (!usage.complete || usage.delta.activeSandboxes === null) { - failures.push('cohort active-resource usage was unavailable') - } else if (usage.delta.activeSandboxes !== 0) { - failures.push(`cohort active-resource delta was ${usage.delta.activeSandboxes}, expected 0`) - } + failures.push(...cohortAccountFailures(attempts, usage)) return [...new Set(failures)] } function finish({ attempts, requestedRuns, concurrency, startedAt, stoppedAfterCanary }) { const usage = cohortUsage(attempts) - const failures = cohortFailures(attempts, requestedRuns, usage) + const failures = cohortFailures(attempts, requestedRuns, concurrency, usage) const activeResourceDeltas = attempts.map( (attempt) => attempt.proof?.cleanup?.activeResourceDelta ?? null, ) + const remainingResourceCounts = attempts.map((attempt) => { + const remainingIds = attempt.proof?.cleanup?.identity?.remainingIds + return Array.isArray(remainingIds) ? remainingIds.length : null + }) return { schemaVersion: 'braid.tangle-sandbox-braid-soak.v1', status: failures.length === 0 ? 'passed' : 'failed', @@ -189,14 +810,10 @@ function finish({ attempts, requestedRuns, concurrency, startedAt, stoppedAfterC cleanup: { exactProofs: attempts.filter((attempt) => attempt.proof?.cleanup?.exactResource === true) .length, - exactResourcesRemaining: attempts.reduce( - (total, attempt) => - total + - (Array.isArray(attempt.proof?.cleanup?.identity?.remainingIds) - ? attempt.proof.cleanup.identity.remainingIds.length - : 1), - 0, - ), + exactResourcesRemaining: remainingResourceCounts.includes(null) + ? null + : remainingResourceCounts.reduce((total, count) => total + count, 0), + resourceProofsUnavailable: remainingResourceCounts.filter((count) => count === null).length, activeResourceDelta: usage.delta.activeSandboxes, activeResourceDeltas, }, @@ -267,13 +884,13 @@ export async function runBraidSandboxSoak({ const canary = await attempt(0, true) attempts.push(canary) - if (!proofPassed(canary.proof) || requestedRuns === 1) { + if (!proofPassed(canary.proof, requestedConcurrency) || requestedRuns === 1) { return finish({ attempts, requestedRuns, concurrency: requestedConcurrency, startedAt, - stoppedAfterCanary: !proofPassed(canary.proof), + stoppedAfterCanary: !proofPassed(canary.proof, requestedConcurrency), }) } @@ -288,7 +905,7 @@ export async function runBraidSandboxSoak({ if (index >= requestedRuns) return const completed = await attempt(index, false) attempts.push(completed) - if (!proofPassed(completed.proof)) stop = true + if (!proofPassed(completed.proof, requestedConcurrency)) stop = true } }, ) diff --git a/scripts/live-required/tangle-sandbox-braid-stress-support.mjs b/scripts/live-required/tangle-sandbox-braid-stress-support.mjs index 2063fa2..f706942 100644 --- a/scripts/live-required/tangle-sandbox-braid-stress-support.mjs +++ b/scripts/live-required/tangle-sandbox-braid-stress-support.mjs @@ -7,6 +7,7 @@ import { requestBase, responseForRequest, runFromState, + stateForRequest, stateForRun, } from '../live-bridge/protocol.mjs' @@ -191,7 +192,10 @@ export async function rpcRoundTrip(session, command, params = {}, operationId, l const request = { ...requestBase(requestId, command, operationId), params } const started = performance.now() session.send(request) - const response = await session.waitFor(label, responseForRequest(requestId)) + const response = await session.waitFor( + label, + command === 'get_state' ? stateForRequest(requestId) : responseForRequest(requestId), + ) return { request, response, elapsedMs: performance.now() - started } } @@ -233,8 +237,91 @@ function belongsToRun(response, runId) { return observedRunId === runId } +function localControlRefFromResponses(responses, runId) { + for (const response of responses) { + if (!belongsToRun(response, runId)) continue + if (eventParts(response)?.event.kind !== 'run.environment.observed') continue + const controlRef = controlRefFromEvent(response) + if (controlRef !== undefined) return controlRef + } + return undefined +} + +function providerIdentityFromEvent(response, kind, eventId) { + const parts = eventParts(response) + const payload = parts?.payload + const provider = record(payload?.provider) + const sources = [ + ['provider', provider, ['runId', 'providerRunId', 'executionId', 'providerExecutionId']], + ['provider.controlRef', record(provider?.controlRef), ['runId', 'executionId']], + ['payload.controlRef', record(payload?.controlRef), ['runId', 'executionId']], + ['payload.providerControlRef', record(payload?.providerControlRef), ['runId', 'executionId']], + ['payload', payload, ['executionId', 'providerRunId', 'providerExecutionId']], + ] + const fields = { + runId: ['runId', 'providerRunId'], + executionId: ['executionId', 'providerExecutionId'], + } + const identity = {} + for (const [field, aliases] of Object.entries(fields)) { + const observed = [] + for (const [sourceName, source, sourceAliases] of sources) { + if (!source) continue + for (const alias of aliases) { + if (!sourceAliases.includes(alias)) continue + if (Object.hasOwn(source, alias)) observed.push({ sourceName, alias, value: source[alias] }) + } + } + if (observed.length === 0) continue + const values = observed.map((candidate) => nonEmptyString(candidate.value)) + if (values.some((value) => value === undefined)) { + throw new MissingIntegrationError( + `Braid emitted visible ${kind} with an invalid provider ${field}`, + { runId: eventRunId(response), kind, eventId, field, observed }, + ) + } + const uniqueValues = [...new Set(values)] + if (uniqueValues.length !== 1) { + throw new MissingIntegrationError( + `Braid emitted visible ${kind} with conflicting provider ${field} values`, + { runId: eventRunId(response), kind, eventId, field, observed }, + ) + } + identity[field] = uniqueValues[0] + } + return Object.keys(identity).length === 0 ? undefined : identity +} + +function assertProviderIdentity(response, runId, kind, eventId, expectedControlRef) { + const observed = providerIdentityFromEvent(response, kind, eventId) + if (observed === undefined) return + if (expectedControlRef === undefined) { + throw new MissingIntegrationError( + `Braid emitted visible ${kind} with provider identity before exposing the exact local control reference`, + { runId, kind, eventId, observed }, + ) + } + for (const field of ['runId', 'executionId']) { + if (observed[field] === undefined) continue + if (observed[field] !== expectedControlRef[field]) { + throw new MissingIntegrationError( + `Braid emitted visible ${kind} for a foreign provider ${field}`, + { + runId, + kind, + eventId, + field, + expected: expectedControlRef[field], + observed: observed[field], + }, + ) + } + } +} + export function providerEventsForRun(responses, runId) { const events = [] + const expectedControlRef = localControlRefFromResponses(responses, runId) for (const response of responses) { if (!belongsToRun(response, runId, responses)) continue const kind = eventParts(response)?.event.kind @@ -264,6 +351,9 @@ export function providerEventsForRun(responses, runId) { { runId, kind, eventId }, ) } + if (!NON_VISIBLE_KINDS.has(kind)) { + assertProviderIdentity(response, runId, kind, eventId, expectedControlRef) + } const payload = eventParts(response)?.payload events.push({ kind, @@ -296,7 +386,29 @@ export function visibleEventKeys(responses, runId) { return visibleProviderEvents(responses, runId).map((event) => event.eventId) } +function assertNoMissingReplayEvidence(responses, runId, phase) { + const missingSequence = [] + const missingHistory = [] + for (const response of responses) { + if (response?.type !== 'state') continue + const state = record(response.state) + const run = runFromState(state, runId) + if (run?.missingSequence !== undefined && run.missingSequence !== null) { + missingSequence.push(run.missingSequence) + } + if (Array.isArray(state?.missingHistory)) { + missingHistory.push(...state.missingHistory.filter((range) => record(range)?.runId === runId)) + } + } + if (missingSequence.length === 0 && missingHistory.length === 0) return + throw new MissingIntegrationError( + `${phase} cannot prove replay because Braid reported missing provider history`, + { runId, missingSequence, missingHistory }, + ) +} + export function assertUniqueVisibleEvents(responses, runId, phase) { + assertNoMissingReplayEvidence(responses, runId, phase) const events = visibleProviderEvents(responses, runId) const keys = events.map((event) => event.eventId) const duplicates = keys.filter((key, index) => keys.indexOf(key) !== index) @@ -323,6 +435,28 @@ export function assertUniqueVisibleEvents(responses, runId, phase) { return { count: keys.length, keys, events } } +function assertContiguousReplay(responses, runId, acknowledgedSequence, phase) { + const freshEvents = providerEventsForRun(responses, runId).filter( + (event) => event.providerSequence > acknowledgedSequence, + ) + assert.ok( + freshEvents.length > 0, + `${phase} replay exposed no provider sequence after the acknowledged cursor`, + ) + assert.equal( + freshEvents[0].providerSequence, + acknowledgedSequence + 1, + `${phase} replay provider sequences after the cursor were not contiguous`, + ) + for (let index = 1; index < freshEvents.length; index += 1) { + assert.equal( + freshEvents[index].providerSequence, + freshEvents[index - 1].providerSequence + 1, + `${phase} replay provider sequences after the cursor were not contiguous`, + ) + } +} + export function exclusiveResumeIntersection(acknowledged, resumed) { const acknowledgedSet = new Set(acknowledged) return [...new Set(resumed)].filter((eventId) => acknowledgedSet.has(eventId)) @@ -364,6 +498,7 @@ export function assertProviderResumeProgress( 'fresh process replay did not advance beyond the acknowledged provider cursor', ) assert.notEqual(first.cursor, cursor, 'fresh process replay started at the acknowledged cursor') + assertContiguousReplay(resumedResponses, runId, cursorEvent.providerSequence, 'fresh-process') return { acknowledgedSequence: cursorEvent.providerSequence, firstFreshSequence: first.providerSequence, diff --git a/scripts/live-required/tangle-sandbox-braid-stress.mjs b/scripts/live-required/tangle-sandbox-braid-stress.mjs index 2bcf903..f3a4266 100644 --- a/scripts/live-required/tangle-sandbox-braid-stress.mjs +++ b/scripts/live-required/tangle-sandbox-braid-stress.mjs @@ -40,6 +40,7 @@ const scriptPath = fileURLToPath(import.meta.url) const repository = resolve(dirname(scriptPath), '../..') const DEFAULT_TIMEOUT_MS = 180_000 const DEFAULT_HOLD_MS = 30_000 +const DEFAULT_IDLE_TTL_SECONDS = 1_800 const SANDBOX_LIST_PAGE_SIZE = 100 const BRAID_RESOURCE_OWNER = 'braid' const RETAINED_LIFECYCLE = 'retained' @@ -80,12 +81,11 @@ function configurationEnvironment(environment) { return environment } -function sandboxClient(values, environment) { - const apiKey = - values.credentialValue?.trim() || - environment.BRAID_TANGLE_SANDBOX_CLEANUP_API_KEY?.trim() || - environment.BRAID_TANGLE_SANDBOX_API_KEY?.trim() || - environment.TANGLE_API_KEY?.trim() +function sandboxClient(values) { + // Direct verification must use the value installed under this proof's + // generated credential reference. An independent cleanup key could observe + // another account and make identity or resource-delta evidence meaningless. + const apiKey = values.credentialValue?.trim() if (!apiKey) return undefined return new Sandbox({ baseUrl: values.endpoint, apiKey }) } @@ -294,7 +294,7 @@ function diagnosticText(value) { if (text.length === 0) return undefined return text .replace( - /\b(authorization|bearer|token|api[-_]?key|password|passphrase|secret|credential)\b\s*[:=]?\s*(?:bearer\s+)?[^\s,;]+/giu, + /\b(authorization|bearer|token|api[-_ ]*key|password|passphrase|secret|credential)\b\s*[:=]?\s*(?:bearer\s+)?[^\s,;]+/giu, '$1=[redacted]', ) .slice(0, 512) @@ -542,12 +542,6 @@ async function cleanupOwnedRetainedResources(client, { controlRef, firstRunId, o listed.push(exact) } } - if (firstRunId !== undefined && listed.length === 0) { - throw new MissingIntegrationError( - 'Cleanup found no retained Sandbox for the acknowledged Braid run', - { firstRunId, operationId, environmentId: controlRef?.environmentId }, - ) - } const deletions = [] for (const box of listed) deletions.push(await deleteOwnedResource(client, box, predicate)) const remaining = (await listAllSandboxes(client)).filter(predicate) @@ -591,6 +585,19 @@ function publicAccountIdentity(value) { } } +function accountIdentityDigest(value) { + return createHash('sha256').update(`${value.customerId}:${value.billingOwnerId}`).digest('hex') +} + +function publicAccountIdentityEvidence(value) { + return { + identityDigest: accountIdentityDigest(value), + ...(typeof value.billingDelegationAuthorized === 'boolean' + ? { billingDelegationAuthorized: value.billingDelegationAuthorized } + : {}), + } +} + async function accountIdentity(client, phase) { if (!client) return { phase, value: undefined, error: undefined } try { @@ -621,8 +628,7 @@ function assertStableAccountIdentity(records) { ) return { stable: true, - customerId: before.value.customerId, - billingOwnerId: before.value.billingOwnerId, + identityDigest: accountIdentityDigest(before.value), } } @@ -642,8 +648,7 @@ function assertSameAccount(state, run, directIdentity) { 'Sandbox billing owner identity changed', ) return { - customerId: observed.customerId, - billingOwnerId: observed.billingOwnerId, + identityDigest: accountIdentityDigest(observed), usage: { computeMinutes: observed.computeMinutes ?? null, gpuSeconds: observed.gpuSeconds ?? null, @@ -671,6 +676,21 @@ function toolEvidence(observation, label) { return events } +export function assertExactRemoteStatus(status, controlRef, label = 'remote execution') { + if (!status) { + throw new MissingIntegrationError(`${label} status was unavailable`, { + executionId: controlRef.executionId, + }) + } + assert.equal( + status.latestExecutionId, + controlRef.executionId, + `${label} status described another execution`, + ) + assertSameControlRef(controlRef, status.runControlRef, `${label} status`) + return status +} + async function verifyRemoteCancellation(client, controlRef, marker, timeoutMs) { const box = await retainedBox(client, controlRef, 'Remote cancellation verification') const session = box.session(controlRef.sessionId) @@ -678,7 +698,13 @@ async function verifyRemoteCancellation(client, controlRef, marker, timeoutMs) { const samples = [] for (;;) { const status = await session.status() - samples.push({ status: status?.status ?? null, observedAt: new Date().toISOString() }) + assertExactRemoteStatus(status, controlRef, 'remote cancellation') + samples.push({ + status: status.status, + activeExecutionId: status.activeExecutionId ?? null, + latestExecutionId: status.latestExecutionId, + observedAt: new Date().toISOString(), + }) if (status?.status === 'cancelled') break if (performance.now() >= deadline) { throw new MissingIntegrationError('The exact cloud execution did not become cancelled', { @@ -689,11 +715,19 @@ async function verifyRemoteCancellation(client, controlRef, marker, timeoutMs) { await sleep(Math.min(250, Math.max(25, deadline - performance.now()))) } await sleep(1_000) - const [settledStatus, messages] = await Promise.all([ + const [settledStatus, exactResult, messages] = await Promise.all([ session.status(), + session.result({ executionId: controlRef.executionId }), box.messages({ sessionId: controlRef.sessionId, limit: 100 }), ]) + assertExactRemoteStatus(settledStatus, controlRef, 'settled remote cancellation') assert.equal(settledStatus?.status, 'cancelled', 'remote cancellation did not remain terminal') + assert.equal( + exactResult.executionId, + controlRef.executionId, + 'remote cancellation result described another execution', + ) + assert.equal(exactResult.success, false, 'cancelled remote execution reported success') const lateResult = messages .filter((message) => message?.role === 'assistant') .some((message) => JSON.stringify(message).includes(marker)) @@ -702,6 +736,12 @@ async function verifyRemoteCancellation(client, controlRef, marker, timeoutMs) { controlRef, samples, settledStatus: settledStatus?.status ?? null, + exactResult: { + executionId: exactResult.executionId, + status: exactResult.status, + success: exactResult.success, + ...(typeof exactResult.errorCode === 'string' ? { errorCode: exactResult.errorCode } : {}), + }, messageCount: messages.length, lateResult, } @@ -781,7 +821,13 @@ export function spendDisclosure(runs) { } } -function telemetryDisclosure(run, state, workspaceVerification, account) { +export function telemetryDisclosure( + run, + state, + workspaceVerification, + account, + { allowInFlight = false } = {}, +) { const environment = state.environments?.find((candidate) => candidate.id === run?.environmentId) const unavailable = environment?.unavailableTelemetry ?? [] const unavailableByPrefix = (prefix) => unavailable.some((entry) => entry.startsWith(prefix)) @@ -790,13 +836,28 @@ function telemetryDisclosure(run, state, workspaceVerification, account) { tokens: run?.tokensKnown === false ? { status: 'unavailable' } - : { status: 'observed', input: run?.inputTokens, output: run?.outputTokens }, + : Number.isSafeInteger(run?.inputTokens) && + run.inputTokens >= 0 && + Number.isSafeInteger(run?.outputTokens) && + run.outputTokens >= 0 + ? { status: 'observed', input: run.inputTokens, output: run.outputTokens } + : allowInFlight + ? { status: 'in-flight' } + : { status: 'missing' }, cost: - run?.usdKnown === false || run?.costUsd === undefined + run?.usdKnown === false ? { status: 'unavailable' } - : { status: 'observed', usd: run.costUsd }, + : typeof run?.costUsd === 'number' && Number.isFinite(run.costUsd) && run.costUsd >= 0 + ? { status: 'observed', usd: run.costUsd } + : allowInFlight + ? { status: 'in-flight' } + : { status: 'missing' }, endToEndDuration: - elapsed === undefined ? { status: 'missing' } : { status: 'observed', milliseconds: elapsed }, + elapsed === undefined + ? allowInFlight + ? { status: 'in-flight' } + : { status: 'missing' } + : { status: 'observed', milliseconds: elapsed }, model: typeof run?.model === 'string' ? { status: 'observed', value: run.model } @@ -829,7 +890,10 @@ function telemetryDisclosure(run, state, workspaceVerification, account) { account: { status: 'observed', value: account }, } const missing = Object.entries(fields) - .filter(([, value]) => value.status === 'missing') + .filter( + ([name, value]) => + value.status === 'missing' || (name === 'resourceSample' && value?.status === 'missing'), + ) .map(([name]) => name) if (missing.length > 0) { throw new MissingIntegrationError('Required Sandbox telemetry was silently missing', { @@ -840,14 +904,57 @@ function telemetryDisclosure(run, state, workspaceVerification, account) { return { completeDisclosure: true, fields, unavailable } } +export function assertVerifiedProcessCleanup(result, label) { + if ( + result?.termination?.exited !== true || + result.termination.descendantsVerified !== true || + result?.exit?.timeout === true + ) { + throw new MissingIntegrationError(`${label} did not prove complete process-tree cleanup`, { + termination: result?.termination, + exit: result?.exit, + }) + } + return { + cleanupStatus: result.termination.cleanupStatus, + exited: true, + descendantsVerified: true, + exit: result.exit, + } +} + +export async function closeBraidWithProof(session, label) { + let shutdownError + try { + await closeSession(session) + } catch (error) { + shutdownError = error + } + const cleanup = assertVerifiedProcessCleanup(await session.close(), label) + if (shutdownError !== undefined) { + if ( + shutdownError !== null && + (typeof shutdownError === 'object' || typeof shutdownError === 'function') + ) { + Object.defineProperty(shutdownError, 'processCleanup', { + configurable: true, + enumerable: false, + value: cleanup, + }) + } + throw shutdownError + } + return cleanup +} + async function killFirstBraid(session) { if (!session?.child) throw new Error('The first Braid process was not started') const sent = session.child.kill('SIGKILL') const exit = await session.exit - await session.close().catch(() => undefined) + const cleanup = assertVerifiedProcessCleanup(await session.close(), 'SIGKILL Braid process') assert.equal(sent, true, 'SIGKILL was not sent to the first Braid process') assert.equal(exit.signal, 'SIGKILL', `first Braid process exited with ${JSON.stringify(exit)}`) - return { signal: exit.signal, code: exit.code, sent } + return { signal: exit.signal, code: exit.code, sent, cleanup } } function assertAck(result, command) { @@ -858,6 +965,16 @@ function assertAck(result, command) { return result.response } +export function assertRestartedCancellationRun(run, controlRef) { + assert.ok(run, 'fresh cancellation process did not restore the cancelled run') + assert.ok( + ['aborted', 'cancelled'].includes(run.status), + `fresh cancellation process restored ${run.status ?? 'missing'}`, + ) + assertSameControlRef(controlRef, run.controlRef, 'cancellation restart') + return run +} + function localRunCount(state, runId) { return (state?.runs ?? []).filter((run) => run.id === runId).length } @@ -868,6 +985,17 @@ function assistantMarker(state, runId) { .find((message) => message.runId === runId && message.role === 'assistant')?.text } +export function runIdForOperation(state, operationId) { + const matches = (state?.runs ?? []).filter((run) => run.operationId === operationId) + if (matches.length > 1) { + throw new MissingIntegrationError( + 'Durable cleanup found more than one Braid run for one operation identity', + { operationId, runIds: matches.map((run) => run.id) }, + ) + } + return matches[0]?.id +} + function collectIntegrationNeed(error, needs) { if (error instanceof MissingIntegrationError) { const fields = (value) => (Array.isArray(value) ? value : value === undefined ? [] : [value]) @@ -899,6 +1027,14 @@ export async function runBraidSandboxStress({ 'BRAID_LIVE_REQUIRED_TIMEOUT_MS', DEFAULT_TIMEOUT_MS, ) + const idleTtlSeconds = numberEnvironment( + environment, + 'BRAID_TANGLE_SANDBOX_IDLE_TTL_SECONDS', + DEFAULT_IDLE_TTL_SECONDS, + ) + if (!Number.isSafeInteger(idleTtlSeconds) || idleTtlSeconds < 60 || idleTtlSeconds > 604_800) { + throw new Error('BRAID_TANGLE_SANDBOX_IDLE_TTL_SECONDS must be an integer from 60 to 604800') + } const prompts = proofPrompts(coordinates, holdMs) const phases = {} const unresolvedIntegrationNeeds = [] @@ -910,6 +1046,7 @@ export async function runBraidSandboxStress({ let client let firstSession let freshSession + let retrySession let firstRunId let cancelRunId let knownEnvironmentId @@ -924,6 +1061,8 @@ export async function runBraidSandboxStress({ let telemetry let followUpVisible let remoteCancellation + let cancelledProcessCleanup + let retryProcessCleanup let resumeFromCursor let finalCursor let resumeIntersection @@ -940,6 +1079,9 @@ export async function runBraidSandboxStress({ let finalUsage let result let binarySha256 + let firstSendAttempted = false + let cleanupRecovery + const failureProcessCleanup = {} const phase = async (name, task) => { const start = performance.now() @@ -951,7 +1093,16 @@ export async function runBraidSandboxStress({ } try { - client = sandboxClient(values, environment) + client = sandboxClient(values) + if (!client) { + throw new MissingIntegrationError( + 'Retained stress requires the same credential value that Braid installs for execution', + { + required: + 'BRAID_TANGLE_SANDBOX_API_KEY or TANGLE_API_KEY; an independent cleanup credential is not admissible proof', + }, + ) + } usageRecords.push(await usage(client, 'before')) identityRecords.push(await accountIdentity(client, 'before')) binary = suppliedBinary ?? (await resolveBinary(suppliedRepository, environment)) @@ -961,12 +1112,14 @@ export async function runBraidSandboxStress({ repository: suppliedRepository, environment: sanitizedEnvironment(environment), ...values, + providerOptions: { lifecycle: RETAINED_LIFECYCLE, idleTtlSeconds }, }), ) const first = await phase('firstProcess.initialize', () => initializedSession(binary, config)) firstSession = first.session const initialState = first.state.state + firstSendAttempted = true const send = await phase('firstProcess.send', () => rpcRoundTrip( firstSession, @@ -1170,13 +1323,6 @@ export async function runBraidSandboxStress({ followUpTerminal.run, identityRecords[0]?.value, ) - telemetry = telemetryDisclosure( - followUpTerminal.run, - followUpTerminal.response.state, - workspaceVerification, - account, - ) - const cancelSend = await phase('cancel.send', () => rpcRoundTrip( freshSession, @@ -1232,9 +1378,19 @@ export async function runBraidSandboxStress({ ), ) + cancelledProcessCleanup = await phase('cancel.restart.closeFirstProcess', () => + closeBraidWithProof(freshSession, 'pre-retry cancellation process'), + ) + freshSession = undefined + const retry = await phase('cancel.restart.initialize', () => initializedSession(binary, config)) + retrySession = retry.session + const retryState = await stateRoundTrip(retrySession) + const retryRun = retryState.state.runs?.find((run) => run.id === cancelRunId) + assertRestartedCancellationRun(retryRun, cancelObservation.controlRef) + const cancelRetry = await phase('cancel.retrySameBody', () => rpcRoundTrip( - freshSession, + retrySession, 'cancel_run', { runId: cancelRunId, reason: `Braid live cancellation ${coordinates.proofId}` }, ids.cancel, @@ -1255,7 +1411,7 @@ export async function runBraidSandboxStress({ const cancelConflict = await phase('cancel.retryChangedBody', () => rpcRoundTrip( - freshSession, + retrySession, 'cancel_run', { runId: cancelRunId, reason: `changed-body-${coordinates.proofId}` }, ids.cancel, @@ -1264,6 +1420,45 @@ export async function runBraidSandboxStress({ ) assert.equal(cancelConflict.response.type, 'error', 'changed cancel body was accepted') assert.equal(cancelConflict.response.code, 'OPERATION_CONFLICT') + retryProcessCleanup = await phase('cancel.restart.closeRetryProcess', () => + closeBraidWithProof(retrySession, 'cancellation retry process'), + ) + retrySession = undefined + + const firstTelemetry = telemetryDisclosure( + firstRun, + firstState.state, + workspaceVerification, + assertSameAccount(firstState.state, firstRun, identityRecords[0]?.value), + { allowInFlight: true }, + ) + const resumedTelemetry = telemetryDisclosure( + freshRun, + freshTerminal.response.state, + workspaceVerification, + assertSameAccount(freshTerminal.response.state, freshRun, identityRecords[0]?.value), + ) + const followUpTelemetry = telemetryDisclosure( + followUpTerminal.run, + followUpTerminal.response.state, + workspaceVerification, + account, + ) + const cancelledTelemetry = telemetryDisclosure( + cancelled.run, + cancelled.response.state, + workspaceVerification, + assertSameAccount(cancelled.response.state, cancelled.run, identityRecords[0]?.value), + ) + telemetry = { + ...followUpTelemetry, + runs: { + first: firstTelemetry, + resumed: resumedTelemetry, + followUp: followUpTelemetry, + cancelled: cancelledTelemetry, + }, + } const runRecords = { first: firstSnapshot, @@ -1288,6 +1483,8 @@ export async function runBraidSandboxStress({ config: configEvidence(config), processes: { first: killed, + cancelled: cancelledProcessCleanup, + retry: retryProcessCleanup, localRunCountAfterReconnect: firstRunCountAfterReplay, binarySha256, }, @@ -1332,8 +1529,79 @@ export async function runBraidSandboxStress({ } collectIntegrationNeed(error, unresolvedIntegrationNeeds) } finally { - if (freshSession) await closeSession(freshSession).catch((error) => (cleanupError ??= error)) - if (firstSession) await firstSession.close().catch((error) => (cleanupError ??= error)) + for (const [label, session] of [ + ['retry', retrySession], + ['fresh', freshSession], + ['first', firstSession], + ]) { + if (!session) continue + try { + failureProcessCleanup[label] = await closeBraidWithProof( + session, + `${label} failure-path Braid process`, + ) + } catch (error) { + if (error?.processCleanup) failureProcessCleanup[label] = error.processCleanup + cleanupError ??= error + collectIntegrationNeed(error, unresolvedIntegrationNeeds) + } + } + if (firstRunId === undefined && firstSendAttempted && binary && config) { + let recoverySession + try { + const recovered = await initializedSession(binary, config) + recoverySession = recovered.session + firstRunId = runIdForOperation(recovered.state.state, ids.first) + const processCleanup = await closeBraidWithProof( + recoverySession, + 'cleanup identity recovery Braid process', + ) + recoverySession = undefined + cleanupRecovery = { + attempted: true, + operationId: ids.first, + runId: firstRunId ?? null, + processCleanup, + } + } catch (error) { + if (error?.processCleanup) { + cleanupRecovery = { + attempted: true, + operationId: ids.first, + runId: firstRunId ?? null, + processCleanup: error.processCleanup, + } + } + cleanupError ??= error + collectIntegrationNeed(error, unresolvedIntegrationNeeds) + } finally { + if (recoverySession) { + try { + const processCleanup = await closeBraidWithProof( + recoverySession, + 'failed cleanup identity recovery Braid process', + ) + cleanupRecovery = { + attempted: true, + operationId: ids.first, + runId: firstRunId ?? null, + processCleanup, + } + } catch (error) { + if (error?.processCleanup) { + cleanupRecovery = { + attempted: true, + operationId: ids.first, + runId: firstRunId ?? null, + processCleanup: error.processCleanup, + } + } + cleanupError ??= error + collectIntegrationNeed(error, unresolvedIntegrationNeeds) + } + } + } + } if (client) { try { const cleanup = await cleanupOwnedRetainedResources(client, { @@ -1461,7 +1729,7 @@ export async function runBraidSandboxStress({ accountIdentities: identityRecords.map((entry) => ({ phase: entry.phase, status: entry.error ? 'unavailable' : entry.value === undefined ? 'missing' : 'observed', - ...(entry.value === undefined ? {} : { value: entry.value }), + ...(entry.value === undefined ? {} : { value: publicAccountIdentityEvidence(entry.value) }), ...(entry.error ? { error: entry.error } : {}), })), accountIdentityConsistency: accountIdentityConsistency ?? null, @@ -1480,6 +1748,8 @@ export async function runBraidSandboxStress({ ...(diagnostics && Object.keys(diagnostics).length > 0 ? { diagnostics } : {}), ...(cleanupError ? { cleanupFailure: errorDetails(cleanupError) } : {}), ...(accountIdentityError ? { accountIdentityFailure: errorDetails(accountIdentityError) } : {}), + ...(cleanupRecovery === undefined ? {} : { cleanupRecovery }), + ...(Object.keys(failureProcessCleanup).length === 0 ? {} : { failureProcessCleanup }), ...(unresolvedIntegrationNeeds.length > 0 ? { unresolvedIntegrationNeeds: [...new Set(unresolvedIntegrationNeeds)] } : {}), diff --git a/scripts/release/redaction.mjs b/scripts/release/redaction.mjs index d096855..6c9e16a 100644 --- a/scripts/release/redaction.mjs +++ b/scripts/release/redaction.mjs @@ -2,15 +2,15 @@ import { createHash } from 'node:crypto' import { StringDecoder } from 'node:string_decoder' const SENSITIVE_FLAG = - /(?:auth|api[-_]?key|bearer|credential|password|private[-_]?key|secret|token)/iu + /(?:auth|api[-_ ]*key|bearer|credential|password|private[-_ ]*key|secret|token)/iu const SECRET_ASSIGNMENT = - /((?:authorization|cookie|credential|api[-_]?key|password|private[-_]?key|secret|token)\s*[:=]\s*)(["']?)[^\s,;}"']+\2/giu + /((?:authorization|cookie|credential|api[-_ ]*key|password|private[-_ ]*key|secret|token)\s*[:=]\s*)(["']?)[^\s,;}"']+\2/giu const BEARER = /\bBearer\s+[A-Za-z0-9._~+/=-]+/giu const URL_USERINFO = /(https?:\/\/)([^\s/@:]+)(?::[^\s/@]*)?@/giu const URL_QUERY_SECRET = /([?&](?:authorization|cookie|credential|api[-_]?key|password|private[-_]?key|secret|token)[^=]*=)[^&#\s]+/giu const ASSIGNMENT_START = - /(?:authorization|cookie|credential|api[-_]?key|password|private[-_]?key|secret|token)\s*[:=]\s*["']?/giu + /(?:authorization|cookie|credential|api[-_ ]*key|password|private[-_ ]*key|secret|token)\s*[:=]\s*["']?/giu const BEARER_START = /\bBearer\s+/giu const URL_USERINFO_START = /https?:\/\/[^\s/@:]+(?::[^\s/@]*)?/giu const URL_QUERY_START = diff --git a/scripts/release/release-collector.test.mjs b/scripts/release/release-collector.test.mjs index cddfc63..c71672e 100644 --- a/scripts/release/release-collector.test.mjs +++ b/scripts/release/release-collector.test.mjs @@ -270,6 +270,7 @@ test('environment sanitization unions explicit and innocent-name canaries withou [...secrets, 'bearer-canary'], ) for (const canary of ['password', 'query-canary', 'bearer-canary']) assert(!text.includes(canary)) + assert.equal(redactText('Invalid API key: phrase-canary'), 'Invalid API key: [REDACTED]') }) test('low-entropy control values stay redacted without corrupting structured release markers', async () => { diff --git a/src/adapters/connections/production-connection-providers.ts b/src/adapters/connections/production-connection-providers.ts index b41f0cf..3c3e45d 100644 --- a/src/adapters/connections/production-connection-providers.ts +++ b/src/adapters/connections/production-connection-providers.ts @@ -67,9 +67,14 @@ export async function capabilitiesForConnection( respondToInteraction: false, }) case 'tangle-sandbox': { - const environment = options.sandboxClient + const reported = options.sandboxClient ? await createTangleProvider({ client: options.sandboxClient }).capabilities() : defaultTangleSandboxCapabilities() + const environment = tangleConnectionCapabilities( + record, + reported, + options.tangleRetainedControlLookup !== undefined, + ) const client = options.sandboxClient return capabilityReport(record, 'executor', environment, { create: true, @@ -81,6 +86,41 @@ export async function capabilitiesForConnection( } } +function tangleConnectionCapabilities( + record: ConnectionRecord, + reported: AgentEnvironmentCapabilities, + hasRetainedControlLookup = false, +): AgentEnvironmentCapabilities { + if (record.providerOptions.lifecycle !== 'retained') { + return { + ...reported, + streaming: { ...reported.streaming, replay: false, detach: false }, + sessions: { ...reported.sessions, continue: false, list: false, messages: false }, + } + } + const retained = reported.retainedControl + const exactRetained = + hasRetainedControlLookup && + retained?.exactRunIdentity === true && + retained.resultIdentity === true && + retained.eventIdentity === true && + retained.cancellationIdempotency === true && + reported.streaming.replay && + reported.streaming.detach && + reported.streaming.turnIdempotency + return { + ...reported, + streaming: { + ...reported.streaming, + live: exactRetained && reported.streaming.live, + replay: exactRetained && reported.streaming.replay, + detach: exactRetained && reported.streaming.detach, + turnIdempotency: exactRetained && reported.streaming.turnIdempotency, + }, + sessions: { ...reported.sessions, continue: false, list: false, messages: false }, + } +} + function capabilityReport( record: ConnectionRecord, backend: ConnectionRuntimeCapabilities['backend'], diff --git a/src/adapters/connections/production-connection-types.ts b/src/adapters/connections/production-connection-types.ts index 56922f0..fd3be4e 100644 --- a/src/adapters/connections/production-connection-types.ts +++ b/src/adapters/connections/production-connection-types.ts @@ -1,4 +1,4 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' +import type { AgentExactRunControlRef, AgentProfile } from '@tangle-network/agent-interface' import type { AgentEnvironmentCapabilities } from '@tangle-network/agent-interface/environment-provider' import type { SandboxClientLike } from '@tangle-network/agent-provider-tangle' import type { RouterTransportConfig } from '@tangle-network/agent-runtime/kernel' @@ -73,6 +73,20 @@ export type SandboxClientFactory = ( input: SandboxClientFactoryInput, ) => SandboxClientLike | Promise +export interface TangleRetainedControlLookupInput { + readonly connectionId: ConnectionId + readonly braidRunId: string + readonly providerSessionId: string + readonly executionId: string + readonly environmentIdempotencyKey: string + readonly signal?: AbortSignal +} + +/** Provider-backed lookup for a dispatch whose acknowledgement was not journaled. */ +export type TangleRetainedControlLookup = ( + input: TangleRetainedControlLookupInput, +) => Promise + export interface ProductionConnectionOptions { readonly credentials?: CredentialPort /** Map Braid's durable credential id to the credential-port's opaque ref. */ @@ -89,6 +103,7 @@ export interface ProductionConnectionOptions { }) => boolean readonly sandboxClient?: SandboxClientLike readonly sandboxClientFactory?: SandboxClientFactory + readonly tangleRetainedControlLookup?: TangleRetainedControlLookup readonly now?: () => IsoDateTime } diff --git a/src/adapters/connections/production-connections.ts b/src/adapters/connections/production-connections.ts index b6d3542..d50eee9 100644 --- a/src/adapters/connections/production-connections.ts +++ b/src/adapters/connections/production-connections.ts @@ -27,6 +27,8 @@ export { type ProductionConnectionOptions, type SandboxClientFactory, type SandboxClientFactoryInput, + type TangleRetainedControlLookup, + type TangleRetainedControlLookupInput, } from './production-connection-types.js' export function createProductionConnectionAdapter( diff --git a/src/adapters/runtime/prepared-execution.ts b/src/adapters/runtime/prepared-execution.ts index 6f7560b..567e35f 100644 --- a/src/adapters/runtime/prepared-execution.ts +++ b/src/adapters/runtime/prepared-execution.ts @@ -6,6 +6,7 @@ export interface SandboxLifecyclePolicy { readonly mode: 'ephemeral' | 'retained' readonly cleanup: 'delete-after-turn' | 'explicit' readonly continuity: 'session' | 'unavailable' + readonly idleTtlSeconds?: number readonly reason?: string } diff --git a/src/adapters/runtime/production-backend-resolver.ts b/src/adapters/runtime/production-backend-resolver.ts index 47cffab..5131f9d 100644 --- a/src/adapters/runtime/production-backend-resolver.ts +++ b/src/adapters/runtime/production-backend-resolver.ts @@ -11,7 +11,11 @@ import { resolveCliBridgeBackend, } from './production-cli-bridge-backend.js' import { resolveTangleInferenceBackend } from './production-tangle-inference-backend.js' -import { resolveTangleSandboxBackend } from './production-tangle-sandbox-backend.js' +import { + type PreparedTangleRetainedConnection, + resolveTangleSandboxBackend, + resolveTangleSandboxRetainedConnection, +} from './production-tangle-sandbox-backend.js' export type { ProductionBackendResolverOptions, @@ -84,3 +88,20 @@ export async function resolveProductionCliBridgeConnection( connectionEndpoint(record, options), ) } + +/** Resolve one retained Tangle plan without creating a billable environment. */ +export async function resolveProductionTangleRetainedConnection( + options: ProductionBackendResolverOptions, + input: ExecuteTurnInput, +): Promise { + const selected = await options.select(input) + const selection = + input.connectionId === undefined + ? selected + : { ...selected, connection: { connectionId: input.connectionId } } + const record = options.connections.select(selection.connection).record + if (record.kind !== 'tangle-sandbox') { + throw new Error('The retained Tangle port received another connection kind') + } + return resolveTangleSandboxRetainedConnection(options, input, selection, record.id) +} diff --git a/src/adapters/runtime/production-tangle-sandbox-backend.ts b/src/adapters/runtime/production-tangle-sandbox-backend.ts index c30b2a3..9079e08 100644 --- a/src/adapters/runtime/production-tangle-sandbox-backend.ts +++ b/src/adapters/runtime/production-tangle-sandbox-backend.ts @@ -1,5 +1,8 @@ -import type { AgentProfile } from '@tangle-network/agent-interface' -import type { AgentEnvironmentCapabilities } from '@tangle-network/agent-interface/environment-provider' +import type { AgentExactRunControlRef, AgentProfile } from '@tangle-network/agent-interface' +import type { + AgentEnvironmentCapabilities, + AgentEnvironmentProvider, +} from '@tangle-network/agent-interface/environment-provider' import type { SandboxClientLike } from '@tangle-network/agent-provider-tangle' import type { SandboxClient } from '@tangle-network/agent-runtime/kernel' import type { BackendType, CreateSandboxOptions, SandboxInstance } from '@tangle-network/sandbox' @@ -8,7 +11,11 @@ import { canonicalDigest } from '../../domain/canonical.js' import type { ConnectionId } from '../../domain/ids.js' import type { ExecuteTurnInput } from '../../ports/execution.js' import { harnessSupportsModel, snapHarnessToModel } from '../agent-interface/harness-runtime.js' -import type { PreparedExecution, SandboxLifecyclePolicy } from './prepared-execution.js' +import type { + ExecutionObservationSource, + PreparedExecution, + SandboxLifecyclePolicy, +} from './prepared-execution.js' import { connectionRecord, exactExecutionProfile, @@ -21,6 +28,11 @@ import { } from './production-backend-common.js' import { observeSandboxClient } from './sandbox-observation.js' import { withSandboxResultProjection } from './sandbox-result-projection.js' +import { + retainedSandboxIdentity, + retainedSandboxLifecycle, + withRetainedSandboxPolicy, +} from './tangle-sandbox-retention.js' export async function resolveTangleSandboxBackend( options: ProductionBackendResolverOptions, @@ -28,28 +40,22 @@ export async function resolveTangleSandboxBackend( selection: ProductionExecutionSelection, connectionId: ConnectionId, ): Promise { - if (input.sessionId !== undefined) { + const record = connectionRecord(connectionId, options) + if (record.providerOptions.lifecycle === 'retained') { throw new ConnectionError( 'CONNECTION_UNSUPPORTED', - 'Tangle sandbox session continuity is unavailable until retained environment recovery is supported', + 'A retained Tangle sandbox connection must execute through the retained execution port', { connectionId }, ) } - const profile = await exactExecutionProfile(input.profile, selection, connectionId) - const model = requiredProfileModel(profile, connectionId) - const runner = await sandboxBackendType(profile, connectionId) - if (!harnessSupportsModel(runner, model)) { - const suggestedRunner = snapHarnessToModel(runner, model) - const runnerChoice = - suggestedRunner === runner - ? 'choose a runner compatible with this model' - : `choose runner=${suggestedRunner} to keep model=${model}` + if (input.sessionId !== undefined) { throw new ConnectionError( - 'CONNECTION_MODEL_HARNESS_MISMATCH', - `Profile field harness=${runner} does not support model=${model}. The authored profile was not changed; ${runnerChoice}, or choose a model advertised for runner=${runner}.`, + 'CONNECTION_UNSUPPORTED', + 'Tangle sandbox session continuity is unavailable until retained environment recovery is supported', { connectionId }, ) } + const { profile, model, runner } = await tangleExecutionIdentity(input, selection, connectionId) const [{ createTangleSandboxClient }, { createTangleProvider }, { createExecutor }] = await Promise.all([ @@ -57,7 +63,6 @@ export async function resolveTangleSandboxBackend( import('@tangle-network/agent-provider-tangle'), import('@tangle-network/agent-runtime/kernel'), ]) - const record = connectionRecord(connectionId, options) const lifecycle = sandboxLifecycle() const idempotencyKey = `env-braid-${safeExecutionId(input.runId)}` const environmentRequestDigest = canonicalDigest({ @@ -112,6 +117,152 @@ export async function resolveTangleSandboxBackend( }) } +export interface PreparedTangleRetainedConnection { + readonly profile: Readonly + readonly model: string + readonly runner: BackendType + readonly provider: AgentEnvironmentProvider + readonly capabilities: AgentEnvironmentCapabilities + readonly observation: ExecutionObservationSource + readonly providerSessionId: string + readonly environmentIdempotencyKey: string + readonly environmentName: string + readonly environmentMetadata: Readonly> + readonly idleTtlSeconds: number + readonly discoverControlRef: ( + braidRunId: string, + signal?: AbortSignal, + ) => Promise + readonly materializationReceipt: Readonly> +} + +/** Resolve a retained cloud session without creating the sandbox. */ +export async function resolveTangleSandboxRetainedConnection( + options: ProductionBackendResolverOptions, + input: ExecuteTurnInput, + selection: ProductionExecutionSelection, + connectionId: ConnectionId, +): Promise { + const record = connectionRecord(connectionId, options) + const idleTtlSeconds = record.providerOptions.idleTtlSeconds + if (record.providerOptions.lifecycle !== 'retained' || idleTtlSeconds === undefined) { + throw new ConnectionError( + 'CONNECTION_UNSUPPORTED', + 'The retained Tangle port requires lifecycle=retained and idleTtlSeconds', + { connectionId }, + ) + } + const { profile, model, runner } = await tangleExecutionIdentity(input, selection, connectionId) + const providerSessionId = input.sessionId ?? `session-braid-${safeExecutionId(input.runId)}` + if (!providerSessionId.startsWith('session-braid-')) { + throw new ConnectionError( + 'CONNECTION_UNSUPPORTED', + 'Retained Tangle continuation requires a Braid-owned provider session', + { connectionId }, + ) + } + const identity = retainedSandboxIdentity(providerSessionId) + const retainedControlLookup = options.tangleRetainedControlLookup + if (retainedControlLookup === undefined) { + throw new ConnectionError( + 'CONNECTION_UNSUPPORTED', + 'Retained Tangle execution requires provider-backed lookup after an unacknowledged dispatch', + { connectionId }, + ) + } + const lifecycle = retainedSandboxLifecycle(idleTtlSeconds) + const [{ createTangleSandboxClient }, { createTangleProvider }] = await Promise.all([ + import('../connections/production-connection-providers.js'), + import('@tangle-network/agent-provider-tangle'), + ]) + const rawClient = await createTangleSandboxClient(record, options, input.signal) + if (rawClient.get === undefined) { + throw new ConnectionError( + 'CONNECTION_UNSUPPORTED', + 'Retained Tangle execution requires exact sandbox reconstruction', + { connectionId }, + ) + } + const boundedClient = withRetainedSandboxPolicy(rawClient, idleTtlSeconds) + const observedClient = observeSandboxClient(boundedClient, lifecycle) + const provider = createTangleProvider({ + client: observedClient.client, + defaultBackend: runner, + name: 'tangle-sandbox', + }) + const reportedCapabilities = await provider.capabilities() + const retained = reportedCapabilities.retainedControl + if ( + retained?.exactRunIdentity !== true || + retained.resultIdentity !== true || + retained.eventIdentity !== true || + retained.cancellationIdempotency !== true || + !reportedCapabilities.streaming.replay || + !reportedCapabilities.streaming.detach || + !reportedCapabilities.streaming.turnIdempotency + ) { + throw new ConnectionError( + 'CONNECTION_UNSUPPORTED', + 'The published Tangle provider did not report exact retained-run control', + { connectionId }, + ) + } + const capabilities = withoutNativeTangleContinuation(reportedCapabilities) + const environmentRequestDigest = canonicalDigest({ + kind: 'tangle-retained-environment-request', + idempotencyKey: identity.environmentIdempotencyKey, + name: identity.name, + metadata: identity.metadata, + idleTtlSeconds, + }) + return freezeExecution({ + profile, + model, + runner, + provider, + capabilities, + observation: observedClient.observation, + providerSessionId, + environmentIdempotencyKey: identity.environmentIdempotencyKey, + environmentName: identity.name, + environmentMetadata: identity.metadata, + idleTtlSeconds, + discoverControlRef: (braidRunId, signal) => + retainedControlLookup({ + connectionId, + braidRunId, + providerSessionId, + executionId: safeExecutionId(braidRunId), + environmentIdempotencyKey: identity.environmentIdempotencyKey, + ...(signal === undefined ? {} : { signal }), + }), + materializationReceipt: { + provider: 'tangle-sandbox', + backend: 'environment-provider', + connectionId, + environmentRequestDigest, + lifecycle: lifecycle.mode, + cleanup: lifecycle.cleanup, + continuity: lifecycle.continuity, + idleTtlSeconds, + portableContext: 'unavailable', + model, + runner, + }, + }) +} + +function withoutNativeTangleContinuation( + reported: AgentEnvironmentCapabilities, +): AgentEnvironmentCapabilities { + const capabilities = { ...reported } + delete capabilities.nativeContinuation + return Object.freeze({ + ...capabilities, + sessions: Object.freeze({ ...reported.sessions, continue: false }), + }) +} + function runtimeSandboxClient( client: SandboxClientLike, name: string, @@ -161,6 +312,33 @@ function providerSessionFor( return input.sessionId ?? `session-braid-${safeExecutionId(input.runId)}` } +async function tangleExecutionIdentity( + input: ExecuteTurnInput, + selection: ProductionExecutionSelection, + connectionId: ConnectionId, +): Promise<{ + readonly profile: Readonly + readonly model: string + readonly runner: BackendType +}> { + const profile = await exactExecutionProfile(input.profile, selection, connectionId) + const model = requiredProfileModel(profile, connectionId) + const runner = await sandboxBackendType(profile, connectionId) + if (!harnessSupportsModel(runner, model)) { + const suggestedRunner = snapHarnessToModel(runner, model) + const runnerChoice = + suggestedRunner === runner + ? 'choose a runner compatible with this model' + : `choose runner=${suggestedRunner} to keep model=${model}` + throw new ConnectionError( + 'CONNECTION_MODEL_HARNESS_MISMATCH', + `Profile field harness=${runner} does not support model=${model}. The authored profile was not changed; ${runnerChoice}, or choose a model advertised for runner=${runner}.`, + { connectionId }, + ) + } + return { profile, model, runner } +} + async function sandboxBackendType( profile: Readonly, connectionId: ConnectionId, diff --git a/src/adapters/runtime/retained-execution-contract.ts b/src/adapters/runtime/retained-execution-contract.ts index f770aec..edfab1f 100644 --- a/src/adapters/runtime/retained-execution-contract.ts +++ b/src/adapters/runtime/retained-execution-contract.ts @@ -19,6 +19,8 @@ export interface RetainedResultProjection { /** Provider-specific facts and operations needed by the retained lifecycle. */ export interface RetainedExecutionPlan { readonly providerName: string + /** False when the provider cannot bind status to this exact execution. */ + readonly exactStatus?: boolean /** Unknown until a provider with server-issued ids admits the first run. */ readonly environmentId?: string readonly providerSessionId?: string diff --git a/src/adapters/runtime/retained-execution-projection.ts b/src/adapters/runtime/retained-execution-projection.ts index 3628017..3a98517 100644 --- a/src/adapters/runtime/retained-execution-projection.ts +++ b/src/adapters/runtime/retained-execution-projection.ts @@ -8,7 +8,13 @@ import type { RuntimeEventEnvelope } from '../../domain/runtime-events.js' import type { RunStatus } from '../../domain/state.js' import type { RunCapabilities } from '../../ports/execution.js' -export function retainedCapabilities(environment: AgentEnvironmentCapabilities): RunCapabilities { +export function retainedCapabilities( + environment: AgentEnvironmentCapabilities, + options: { + readonly sessionContinuation?: boolean + readonly exactStatus?: boolean + } = {}, +): RunCapabilities { return Object.freeze({ streaming: { live: environment.streaming.live, @@ -16,8 +22,17 @@ export function retainedCapabilities(environment: AgentEnvironmentCapabilities): detach: environment.streaming.detach, turnIdempotency: environment.streaming.turnIdempotency, }, - sessions: { continue: environment.sessions.continue, messages: false }, - controls: { cancel: true, steer: false, queue: true, status: true, recreate: true }, + sessions: { + continue: options.sessionContinuation ?? environment.sessions.continue, + messages: false, + }, + controls: { + cancel: true, + steer: false, + queue: true, + status: options.exactStatus ?? true, + recreate: true, + }, events: { stableIdentity: true, sequence: true, cursor: true }, usage: environment.usage, environment, diff --git a/src/adapters/runtime/retained-execution.ts b/src/adapters/runtime/retained-execution.ts index 8ec1881..0000919 100644 --- a/src/adapters/runtime/retained-execution.ts +++ b/src/adapters/runtime/retained-execution.ts @@ -72,6 +72,7 @@ export class RetainedExecutionPort implements ExecutionPort { } finally { this.#state.clearStartingHandle(input.runId, starting) } + if (this.#state.isDetached(input.runId)) return this.#state.markDetached(input.runId, false) yield* streamRetainedExecution({ runId: input.runId, @@ -124,12 +125,12 @@ export class RetainedExecutionPort implements ExecutionPort { readonly signal?: AbortSignal }): Promise { this.#validateKnownControl(input.runId, input.providerSessionId, input.controlRef) - const reader = this.#state.reader(input.runId) - this.#state.markDetached(input.runId, true) - if (!reader) { + if (this.#state.isDetached(input.runId)) { return { operationId: input.operationId, outcome: 'already-applied', detail: 'detached' } } - reader.abort(new DOMException('Braid detached from the retained run', 'AbortError')) + const reader = this.#state.reader(input.runId) + this.#state.markDetached(input.runId, true) + reader?.abort(new DOMException('Braid detached from the retained run', 'AbortError')) return { operationId: input.operationId, outcome: 'accepted', detail: 'detached' } } @@ -187,6 +188,9 @@ export class RetainedExecutionPort implements ExecutionPort { } let effect = result.effect if (effect === 'unknown' || effect === 'not_live') { + if (resolved.plan.exactStatus === false) { + return { operationId: input.operationId, outcome: 'unknown', detail: result.effect } + } const reconciled = await this.#reconcileCancelled(resolved, input) if (!reconciled) { return { operationId: input.operationId, outcome: 'unknown', detail: result.effect } @@ -218,6 +222,7 @@ export class RetainedExecutionPort implements ExecutionPort { input.signal, ) if (!resolved) return null + if (resolved.plan.exactStatus === false) return null const snapshot = await resolved.handle.status({ ...(input.signal === undefined ? {} : { signal: input.signal }), }) @@ -281,7 +286,6 @@ export class RetainedExecutionPort implements ExecutionPort { } if (activePlan !== undefined) { if (this.#state.isCancellationRequested(runId)) return null - throw new Error('Retained run state is incomplete') } const recoverySessionId = this.#recoverySessionId(providerSessionId, supplied) const plan = await this.#planFor(runId, recoverySessionId, supplied) diff --git a/src/adapters/runtime/tangle-retained-execution.ts b/src/adapters/runtime/tangle-retained-execution.ts new file mode 100644 index 0000000..44d6f44 --- /dev/null +++ b/src/adapters/runtime/tangle-retained-execution.ts @@ -0,0 +1,33 @@ +import type { AgentExactRunControlRef } from '@tangle-network/agent-interface' +import type { ExecuteTurnInput } from '../../ports/execution.js' +import type { PreparedTangleRetainedConnection } from './production-tangle-sandbox-backend.js' +import { RetainedExecutionPort } from './retained-execution.js' +import { createTangleRetainedPlan } from './tangle-retained-run.js' + +export interface TangleRetainedExecutionOptions { + readonly resolve: (input: ExecuteTurnInput) => Promise + readonly recover: (input: { + readonly runId: string + readonly providerSessionId?: string + readonly controlRef?: AgentExactRunControlRef + }) => Promise +} + +/** The cloud owns the retained sandbox; Braid owns readers and exact recovery state. */ +export class TangleRetainedExecutionPort extends RetainedExecutionPort { + constructor(options: TangleRetainedExecutionOptions) { + super({ + resolve: async (input) => createTangleRetainedPlan(await options.resolve(input), input.runId), + recover: async ({ runId, providerSessionId, controlRef }) => + createTangleRetainedPlan( + await options.recover({ + runId, + ...(providerSessionId === undefined ? {} : { providerSessionId }), + ...(controlRef === undefined ? {} : { controlRef }), + }), + runId, + controlRef, + ), + }) + } +} diff --git a/src/adapters/runtime/tangle-retained-run.ts b/src/adapters/runtime/tangle-retained-run.ts new file mode 100644 index 0000000..8413174 --- /dev/null +++ b/src/adapters/runtime/tangle-retained-run.ts @@ -0,0 +1,111 @@ +import type { AgentExactRunControlRef } from '@tangle-network/agent-interface' +import type { AgentEnvironmentProvider } from '@tangle-network/agent-interface/environment-provider' +import { + type RetainedRunHandle, + reconnectRetainedRun, + startRetainedRun, +} from '@tangle-network/agent-runtime/kernel' +import { publicMaterializationReceipt } from '../../domain/materialization-receipt.js' +import type { RuntimeEventEnvelope } from '../../domain/runtime-events.js' +import type { ExecuteTurnInput } from '../../ports/execution.js' +import type { PreparedTangleRetainedConnection } from './production-tangle-sandbox-backend.js' +import { safeExecutionId } from './production-backend-common.js' +import type { + RetainedExecutionPlan, + RetainedResultProjection, +} from './retained-execution-contract.js' +import { + finalRetainedEnvelope, + isTerminalRetainedStatus, + modelRequestsFromResult, + retainedCapabilities, + retainedStatus, + retainedTurnUsage, +} from './retained-execution-projection.js' + +export interface TangleRetainedPlan extends RetainedExecutionPlan { + readonly prepared: PreparedTangleRetainedConnection + readonly provider: AgentEnvironmentProvider + readonly executionId: string +} + +export function createTangleRetainedPlan( + prepared: PreparedTangleRetainedConnection, + runId: string, + controlRef?: AgentExactRunControlRef, +): TangleRetainedPlan { + const executionId = controlRef?.executionId ?? safeExecutionId(runId) + const plan: TangleRetainedPlan = { + prepared, + provider: prepared.provider, + executionId, + providerName: prepared.provider.name, + exactStatus: false, + ...(controlRef === undefined ? {} : { environmentId: controlRef.environmentId }), + providerSessionId: controlRef?.sessionId ?? prepared.providerSessionId, + model: prepared.model, + capabilities: retainedCapabilities(prepared.capabilities, { + sessionContinuation: false, + exactStatus: false, + }), + materializationReceipt: publicMaterializationReceipt({ + ...prepared.materializationReceipt, + providerRunId: controlRef?.runId ?? executionId, + retainedControl: 'exact-after-dispatch', + }), + start: (input) => startTangleRetainedRun(plan, input), + reconnect: (exact) => reconnectRetainedRun({ provider: plan.provider, controlRef: exact }), + discover: async (braidRunId, signal) => { + const exact = await prepared.discoverControlRef(braidRunId, signal) + if (exact === null) return null + if (exact.executionId !== executionId) { + throw new Error('Tangle retained lookup returned another execution') + } + return exact + }, + observe: () => prepared.observation.snapshot(), + projectStatus: ({ status, detached }) => retainedStatus(status, detached), + isTerminalStatus: isTerminalRetainedStatus, + projectResult: (result): RetainedResultProjection => ({ + text: result.text, + usage: retainedTurnUsage(result.usage, prepared.model, modelRequestsFromResult(result)), + ...(result.error === undefined ? {} : { error: result.error }), + }), + projectFinal: ({ runId: braidRunId, sequence, result }): RuntimeEventEnvelope => + finalRetainedEnvelope( + braidRunId, + sequence, + prepared.model, + result, + 'Execute the retained Tangle sandbox turn', + ), + } + return Object.freeze(plan) +} + +export async function startTangleRetainedRun( + plan: TangleRetainedPlan, + input: ExecuteTurnInput, +): Promise { + // A retained create may return an existing environment for the same key. + // Do not destroy it after an ambiguous dispatch failure without a provider-issued creation receipt. + return startRetainedRun({ + provider: plan.provider, + environment: { + profile: plan.prepared.profile, + backend: plan.prepared.runner, + name: plan.prepared.environmentName, + metadata: plan.prepared.environmentMetadata, + idempotencyKey: plan.prepared.environmentIdempotencyKey, + }, + turn: { + prompt: input.text, + turnId: safeExecutionId(input.operationId), + signal: input.signal, + }, + identity: { + sessionId: plan.prepared.providerSessionId, + executionId: plan.executionId, + }, + }) +} diff --git a/src/adapters/runtime/tangle-sandbox-retention.ts b/src/adapters/runtime/tangle-sandbox-retention.ts new file mode 100644 index 0000000..d73442f --- /dev/null +++ b/src/adapters/runtime/tangle-sandbox-retention.ts @@ -0,0 +1,93 @@ +import type { SandboxClientLike, SandboxInstanceLike } from '@tangle-network/agent-provider-tangle' +import type { CreateSandboxOptions } from '@tangle-network/sandbox' +import { + MAX_RETAINED_IDLE_TTL_SECONDS, + MIN_RETAINED_IDLE_TTL_SECONDS, +} from '../../domain/entities-core.js' +import type { SandboxLifecyclePolicy } from './prepared-execution.js' +import { safeExecutionId } from './production-backend-common.js' +import type { ObservableSandboxClient } from './sandbox-observation-types.js' + +export { MAX_RETAINED_IDLE_TTL_SECONDS, MIN_RETAINED_IDLE_TTL_SECONDS } + +export interface RetainedSandboxIdentity { + readonly providerSessionId: string + readonly environmentIdempotencyKey: string + readonly name: string + readonly metadata: Readonly<{ + owner: 'braid' + lifecycle: 'retained' + providerSessionId: string + }> +} + +export function retainedSandboxIdentity(providerSessionId: string): RetainedSandboxIdentity { + const suffix = safeExecutionId(providerSessionId) + return Object.freeze({ + providerSessionId, + environmentIdempotencyKey: `env-braid-${suffix}`, + name: `braid-${suffix}`, + metadata: Object.freeze({ owner: 'braid', lifecycle: 'retained', providerSessionId }), + }) +} + +export function retainedSandboxLifecycle(idleTtlSeconds: number): SandboxLifecyclePolicy { + return Object.freeze({ + mode: 'retained', + cleanup: 'explicit', + continuity: 'unavailable', + idleTtlSeconds, + reason: 'The deployed provider has not proved retry-safe native session continuation', + }) +} + +/** Apply Braid's bounded lifetime without replacing the Sandbox SDK. */ +export function withRetainedSandboxPolicy( + source: SandboxClientLike, + idleTtlSeconds: number, +): SandboxClientLike { + const observable = source as ObservableSandboxClient + const get = source.get?.bind(source) + const list = source.list?.bind(source) + const describePlacement = source.describePlacement?.bind(source) + const getIdentity = observable.getIdentity?.bind(source) + const usage = observable.usage?.bind(source) + const subscription = observable.subscription?.bind(source) + return Object.freeze({ + async create(options?: CreateSandboxOptions, requestOptions?: { signal?: AbortSignal }) { + const box = await source.create( + { + ...options, + ephemeral: false, + idleTimeoutSeconds: idleTtlSeconds, + }, + requestOptions, + ) + return box + }, + ...(get === undefined + ? {} + : { + async get(id: string, requestOptions?: { signal?: AbortSignal }) { + const box = await get(id, requestOptions) + return box + }, + }), + ...(list === undefined + ? {} + : { + list: (options?: { + scope?: string + limit?: number + offset?: number + signal?: AbortSignal + }) => list(options), + }), + ...(describePlacement === undefined + ? {} + : { describePlacement: (box: SandboxInstanceLike) => describePlacement(box) }), + ...(getIdentity === undefined ? {} : { getIdentity }), + ...(usage === undefined ? {} : { usage }), + ...(subscription === undefined ? {} : { subscription }), + }) +} diff --git a/src/app/connections.ts b/src/app/connections.ts index 5e21c59..c381502 100644 --- a/src/app/connections.ts +++ b/src/app/connections.ts @@ -215,6 +215,7 @@ function assertSecretFree(record: ConnectionRecord): void { record.providerOptions.endpoint, record.providerOptions.region, record.providerOptions.account, + record.providerOptions.lifecycle, ...(record.providerOptions.capabilityHints ?? []), ...('message' in record.lastHealth ? [record.lastHealth.message] : []), ...(record.lastModelVerification === undefined diff --git a/src/app/production-composition.ts b/src/app/production-composition.ts index decd5a1..e4ee309 100644 --- a/src/app/production-composition.ts +++ b/src/app/production-composition.ts @@ -10,7 +10,9 @@ import { createProductionBackendResolver, type ProductionBackendResolverOptions, resolveProductionCliBridgeConnection, + resolveProductionTangleRetainedConnection, } from '../adapters/runtime/production-backend-resolver.js' +import { TangleRetainedExecutionPort } from '../adapters/runtime/tangle-retained-execution.js' import type { ConnectionRecord } from '../domain/entities.js' import type { ExecutionPort } from '../ports/execution.js' import { ConnectionError } from './connection-errors.js' @@ -187,17 +189,32 @@ export function createProductionComposition( ...(config.workspaceRoot === undefined ? {} : { workspaceRoot: config.workspaceRoot }), signal: new AbortController().signal, }) - const execution = - connection.kind === 'cli-bridge' - ? new CliBridgeRetainedExecutionPort({ - resolve: (input) => resolveProductionCliBridgeConnection(resolverOptions, input), - recover: ({ runId, providerSessionId }) => - resolveProductionCliBridgeConnection( - resolverOptions, - recoveryInput(runId, providerSessionId), - ), - }) - : new AgentRuntimeExecutionPort(backendResolver) + const execution = (() => { + if (connection.kind === 'cli-bridge') { + return new CliBridgeRetainedExecutionPort({ + resolve: (input) => resolveProductionCliBridgeConnection(resolverOptions, input), + recover: ({ runId, providerSessionId }) => + resolveProductionCliBridgeConnection( + resolverOptions, + recoveryInput(runId, providerSessionId), + ), + }) + } + if ( + connection.kind === 'tangle-sandbox' && + connection.providerOptions.lifecycle === 'retained' + ) { + return new TangleRetainedExecutionPort({ + resolve: (input) => resolveProductionTangleRetainedConnection(resolverOptions, input), + recover: ({ runId, providerSessionId }) => + resolveProductionTangleRetainedConnection( + resolverOptions, + recoveryInput(runId, providerSessionId), + ), + }) + } + return new AgentRuntimeExecutionPort(backendResolver) + })() return Object.freeze({ profile, diff --git a/src/app/provider-values.ts b/src/app/provider-values.ts index 90c5410..7c03e04 100644 --- a/src/app/provider-values.ts +++ b/src/app/provider-values.ts @@ -15,6 +15,14 @@ export { safePublicIdentifier, } from '../domain/provider-values.js' +function safeProperty(value: object, key: string): unknown { + try { + return Reflect.get(value, key) + } catch { + return undefined + } +} + /** * Provider payloads are untrusted input to durable state. * @@ -26,6 +34,21 @@ export { */ export function safeRuntimeDiagnostic(value: unknown, fallback: string): string { + if (typeof value === 'object' && value !== null) { + const message = safeProviderDiagnostic(safeProperty(value, 'message'), '') + if (message.length > 0) return message + const code = safeProviderDiagnostic(safeProperty(value, 'code'), '') + if (code.length > 0) return code + const name = safeProperty(value, 'name') + if (typeof name === 'string') { + const typedName = name + .replace(/([a-z0-9])([A-Z])/gu, '$1_$2') + .replace(/[^A-Za-z0-9._]/gu, '_') + .toUpperCase() + const safeName = safeProviderDiagnostic(typedName, '') + if (safeName.length > 0 && safeName !== 'ERROR') return safeName + } + } return safeProviderDiagnostic(value, fallback) } diff --git a/src/app/run-execution.ts b/src/app/run-execution.ts index 2ace37f..bfec1a4 100644 --- a/src/app/run-execution.ts +++ b/src/app/run-execution.ts @@ -64,10 +64,7 @@ export async function executeRun( if (!terminalSeen) await finishWithoutTerminal(context, input, admission, abort) } catch (error) { if (context.ledger.isDetached(admission.runId)) return - const message = safeRuntimeDiagnostic( - error instanceof Error ? error.message : String(error), - 'RUNTIME_EXECUTION_ERROR', - ) + const message = safeRuntimeDiagnostic(error, 'RUNTIME_EXECUTION_ERROR') if (terminalSeen) throw error if (!terminalSeen) await finishAfterError(context, input, admission, abort, message) } finally { @@ -157,6 +154,7 @@ async function finishAfterError( await reconnectRun(context, { operationId: `${input.operationId}:reconnect`, runId: admission.runId, + priorFailureDetail: message, }) return } diff --git a/src/app/run-replay.ts b/src/app/run-replay.ts index 2f65691..0c2fd0e 100644 --- a/src/app/run-replay.ts +++ b/src/app/run-replay.ts @@ -5,9 +5,13 @@ import type { ReconnectInput, ReplayPort } from './application-ports.js' import { AppError } from './errors.js' import { safeSnapshotDetail, safeSnapshotText, safeSnapshotUsage } from './provider-snapshot.js' +interface RecoveryReconnectInput extends ReconnectInput { + readonly priorFailureDetail?: string +} + export async function reconnectRun( context: ReplayPort, - input: ReconnectInput, + input: RecoveryReconnectInput, ): Promise { const run = context.findRun(input.runId) if (context.isTerminal(run.status) && run.status !== 'unknown') @@ -43,7 +47,13 @@ export async function reconnectRun( if (result.accepted && envelope.event.type === 'final') sawTerminal = true } if (!sawTerminal && !context.isTerminal(context.findRun(run.id).status)) - await reconcileRun(context, { runId: run.id, operationId: input.operationId }) + await reconcileRun(context, { + runId: run.id, + operationId: input.operationId, + ...(input.priorFailureDetail === undefined + ? {} + : { priorFailureDetail: input.priorFailureDetail }), + }) } catch (error) { if (!context.isTerminal(context.findRun(run.id).status)) await context.commitAndWait({ @@ -60,7 +70,7 @@ export async function reconnectRun( export async function reconcileRun( context: ReplayPort, - input: ReconnectInput, + input: RecoveryReconnectInput, ): Promise { const run = context.findRun(input.runId) if (!run.capabilities.controls.status || !context.execution.status) { @@ -68,7 +78,7 @@ export async function reconcileRun( await context.commitAndWait({ kind: 'run.unknown', runId: run.id, - detail: 'The execution path cannot reconcile provider state', + detail: input.priorFailureDetail ?? 'The execution path cannot reconcile provider state', }) return structuredClone(context.currentState()) } @@ -96,7 +106,7 @@ export async function reconcileRun( await context.commitAndWait({ kind: 'run.unknown', runId: run.id, - detail: 'The provider returned no run record', + detail: input.priorFailureDetail ?? 'The provider returned no run record', }) return structuredClone(context.currentState()) } diff --git a/src/domain/entities-core.ts b/src/domain/entities-core.ts index 85d3ed3..408d6fa 100644 --- a/src/domain/entities-core.ts +++ b/src/domain/entities-core.ts @@ -141,6 +141,10 @@ export interface ConnectionRecord { readonly lastModelVerification?: ConnectionModelVerification } +export const DEFAULT_RETAINED_IDLE_TTL_SECONDS = 1_800 +export const MIN_RETAINED_IDLE_TTL_SECONDS = 60 +export const MAX_RETAINED_IDLE_TTL_SECONDS = 604_800 + /** * Braid stores only transport metadata and capability hints here. * Provider-native configuration belongs to the provider package and is never @@ -152,6 +156,10 @@ export interface ConnectionTransportOptions { readonly region?: string readonly account?: string readonly capabilityHints?: readonly string[] + /** Cloud environment ownership policy. Omitted connections remain ephemeral. */ + readonly lifecycle?: 'ephemeral' | 'retained' + /** Idle expiry for a retained cloud environment. */ + readonly idleTtlSeconds?: number } export interface RetentionPolicy { diff --git a/src/domain/invariants-profile.ts b/src/domain/invariants-profile.ts index 8ae4236..1cc37dc 100644 --- a/src/domain/invariants-profile.ts +++ b/src/domain/invariants-profile.ts @@ -10,6 +10,7 @@ import type { ProfileSnapshotRecord, WorkspaceRecord, } from './entities.js' +import { MAX_RETAINED_IDLE_TTL_SECONDS, MIN_RETAINED_IDLE_TTL_SECONDS } from './entities-core.js' import { assertDate, assertDigest, @@ -86,7 +87,15 @@ export function assertConnectionRecord(record: ConnectionRecord): void { assertEntityId('workspace', record.workspaceId, 'connection.workspaceId') nonEmpty(record.name, 'connection.name') if (record.endpoint !== undefined) assertPublicReference(record.endpoint, 'connection.endpoint') - const allowedOptions = new Set(['transport', 'endpoint', 'region', 'account', 'capabilityHints']) + const allowedOptions = new Set([ + 'transport', + 'endpoint', + 'region', + 'account', + 'capabilityHints', + 'lifecycle', + 'idleTtlSeconds', + ]) for (const [key, value] of Object.entries(record.providerOptions)) { if (!allowedOptions.has(key)) fail(`connection.providerOptions.${key} is provider-native state`) if (key === 'capabilityHints') { @@ -96,12 +105,45 @@ export function assertConnectionRecord(record: ConnectionRecord): void { ) { fail('connection.providerOptions.capabilityHints must contain non-empty names') } + } else if (key === 'lifecycle') { + if (value !== 'ephemeral' && value !== 'retained') { + fail('connection.providerOptions.lifecycle is invalid') + } + } else if (key === 'idleTtlSeconds') { + if ( + !Number.isSafeInteger(value) || + (value as number) < MIN_RETAINED_IDLE_TTL_SECONDS || + (value as number) > MAX_RETAINED_IDLE_TTL_SECONDS + ) { + fail( + `connection.providerOptions.idleTtlSeconds must be an integer from ${MIN_RETAINED_IDLE_TTL_SECONDS} to ${MAX_RETAINED_IDLE_TTL_SECONDS}`, + ) + } } else if (typeof value !== 'string' || value.length === 0) { fail(`connection.providerOptions.${key} must be a non-empty string`) } if (key === 'endpoint' && typeof value === 'string') assertPublicReference(value, 'connection.providerOptions.endpoint') } + if ( + record.providerOptions.lifecycle === 'retained' && + record.providerOptions.idleTtlSeconds === undefined + ) { + fail('connection.providerOptions.lifecycle=retained requires idleTtlSeconds') + } + if ( + record.providerOptions.lifecycle !== 'retained' && + record.providerOptions.idleTtlSeconds !== undefined + ) { + fail('connection.providerOptions.idleTtlSeconds requires lifecycle=retained') + } + if ( + record.kind !== 'tangle-sandbox' && + (record.providerOptions.lifecycle !== undefined || + record.providerOptions.idleTtlSeconds !== undefined) + ) { + fail('connection.providerOptions lifecycle is available only for tangle-sandbox') + } assertDate(record.createdAt, 'connection.createdAt') assertDate(record.updatedAt, 'connection.updatedAt') if (record.credentialRef !== undefined) diff --git a/src/domain/secret-sanitizer.ts b/src/domain/secret-sanitizer.ts index 0d578fc..b8ffcf8 100644 --- a/src/domain/secret-sanitizer.ts +++ b/src/domain/secret-sanitizer.ts @@ -6,7 +6,7 @@ const MAX_PENDING_BYTES = 4096 const MAX_LOOKBEHIND_CHARS = 1024 const BIDI_CONTROLS = /\p{Bidi_Control}/gu const SECRET_ASSIGNMENT = - /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;}\])}]*)/giu + /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_ -]*key|access[_ -]*key|private[_ -]*key|client[_ -]*secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*"|'[^']*'|[^\s,;}\])}]*)/giu const BEARER_ASSIGNMENT = /\bBearer(?:\s+|\s*=\s*)[^\s,;]*/giu const BARE_CREDENTIAL = /(?:sk-[A-Za-z0-9_-]{20,}|github_pat_[A-Za-z0-9_]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{30,}|xox[baprs]-[A-Za-z0-9-]{20,})/gu @@ -14,7 +14,7 @@ const WEB_URL = /https?:\/\/[^\s\p{Cc}<>"']+/giu const INCOMPLETE_BEARER = /\bBearer(?:[\s]+|[\s]*=[\s]*)$/iu const INCOMPLETE_URL = /(^|[\s([{<])https?:\/\/[^\s\p{Cc}<>"']*$/iu const INCOMPLETE_ASSIGNMENT = - /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*|'[^']*|[^\s,;}\])}]*)$/iu + /(^|[\s,;{[(])(?:password|passwd|passphrase|token|secret|credential|authorization|auth|key|api[_ -]*key|access[_ -]*key|private[_ -]*key|client[_ -]*secret|signature|cookie|header|query|fragment)\s*[:=]\s*(?:"[^"]*|'[^']*|[^\s,;}\])}]*)$/iu const INCOMPLETE_BARE_CREDENTIAL = /(?:sk-|github_pat_|gh[pousr]_|AKIA|AIza|xox[baprs]-)[A-Za-z0-9_-]*$/u diff --git a/test/application.test.ts b/test/application.test.ts index 61f0f2b..3d81e65 100644 --- a/test/application.test.ts +++ b/test/application.test.ts @@ -9,6 +9,7 @@ import { AppError, BraidApplication } from '../src/app/application.js' import { createBraidApplication, DETERMINISTIC_PROFILE } from '../src/app/composition.js' import { effectRequestDigest } from '../src/app/effect-coordinator.js' import { MemoryJournal } from '../src/app/journal.js' +import { safeRuntimeDiagnostic } from '../src/app/provider-values.js' import { createProfileRecord } from '../src/app/profiles.js' import { runEffectRequest } from '../src/app/run-admission.js' import { buildAppView } from '../src/app/view-model.js' @@ -813,6 +814,19 @@ test('provider diagnostics and model metadata cannot persist credential material assert.equal(state.runs[0]?.outputTokens, 0) }) +test('provider diagnostic getters cannot break execution failure handling', () => { + const hostile = new Proxy( + {}, + { + get() { + throw new Error('hostile provider getter') + }, + }, + ) + + assert.equal(safeRuntimeDiagnostic(hostile, 'RUNTIME_EXECUTION_ERROR'), 'RUNTIME_EXECUTION_ERROR') +}) + test('subscriber failures cannot alter a completed run', async () => { const app = createBraidApplication({ fixture: 'deterministic' }) app.subscribe(() => { diff --git a/test/domain-invariants.test.ts b/test/domain-invariants.test.ts index 58250f0..7a11883 100644 --- a/test/domain-invariants.test.ts +++ b/test/domain-invariants.test.ts @@ -196,6 +196,14 @@ test('connection state accepts transport metadata but rejects provider-native op }), /credential material|URL credentials/u, ) + assert.throws( + () => + assertConnectionRecord({ + ...connection, + providerOptions: { lifecycle: 'retained', idleTtlSeconds: 1_800 }, + }), + /only for tangle-sandbox/u, + ) }) test('unknown interaction kinds remain renderable through the canonical answer specification', () => { diff --git a/test/domain-text.test.ts b/test/domain-text.test.ts index 229f849..b695b08 100644 --- a/test/domain-text.test.ts +++ b/test/domain-text.test.ts @@ -65,3 +65,16 @@ test('known bare credential formats are removed across every stream boundary', ( expected, ) }) + +test('phrase-form credential assignments are removed across stream boundaries', () => { + const source = 'Provider rejected API key: sk-live-sentinel-1234567890' + const expected = 'Provider rejected [redacted secret]' + assert.equal(redactSensitiveText(source), expected) + for (let boundary = 0; boundary <= source.length; boundary += 1) { + assert.equal( + sanitizeTextChunks([source.slice(0, boundary), source.slice(boundary)]), + expected, + `split at ${boundary}`, + ) + } +}) diff --git a/test/production-connection-actions.test.ts b/test/production-connection-actions.test.ts index c431d0e..3f17b15 100644 --- a/test/production-connection-actions.test.ts +++ b/test/production-connection-actions.test.ts @@ -1135,6 +1135,8 @@ test('retrying the same create operation is idempotent across config, catalog, c const created = fixture.app .state() .connections.find((record) => record.id === first.connection.id) + assert.equal(created?.providerOptions.lifecycle, undefined) + assert.equal(created?.providerOptions.idleTtlSeconds, undefined) assert.ok(created?.credentialRef) if (created?.credentialRef === undefined) throw new Error('retry record has no credential ref') const stored = await fixture.credentials.resolve(portCredentialRef(created.credentialRef)) diff --git a/test/production-connection-setup.test.ts b/test/production-connection-setup.test.ts index 444eb02..6e2478f 100644 --- a/test/production-connection-setup.test.ts +++ b/test/production-connection-setup.test.ts @@ -59,6 +59,8 @@ test('first-run discovery offers CLI Bridge, Tangle inference, and Tangle sandbo ) assert.equal(setup.connections[1]?.endpoint, 'https://router.tangle.tools') assert.equal(setup.connections[2]?.endpoint, 'https://sandbox.tangle.tools') + assert.equal(setup.connections[2]?.providerOptions.lifecycle, undefined) + assert.equal(setup.connections[2]?.providerOptions.idleTtlSeconds, undefined) assert.equal( setup.connections.every((connection) => connection.credentialRef === undefined), true, diff --git a/test/retained-execution-lifecycle.test.ts b/test/retained-execution-lifecycle.test.ts index f1805ca..58efd1b 100644 --- a/test/retained-execution-lifecycle.test.ts +++ b/test/retained-execution-lifecycle.test.ts @@ -203,6 +203,50 @@ test('in-flight start cancellation omits an expired foreground signal', async () assert.equal(receivedSignal, undefined) }) +test('in-flight detach starts the cloud run but never opens a local event reader', async () => { + const exact = controlRef('detach-in-flight') + let resolveStart: (value: RetainedRunHandle) => void = () => undefined + let starts = 0 + const started = new Promise((resolve) => { + resolveStart = resolve + }) + const retainedHandle = handle(exact) + const execution = executionFor(async () => + plan(exact, async () => { + starts += 1 + return started + }), + ) + const runInput = input('detach-in-flight') + await execution.admit(runInput) + const stream = execution.streamTurn(runInput)[Symbol.asyncIterator]() + const streamCompletion = stream.next() + await new Promise((resolve) => setImmediate(resolve)) + + const acknowledgement = await execution.detachRun({ + operationId: 'operation-detach-in-flight', + runId: runInput.runId, + }) + resolveStart(retainedHandle) + + assert.deepEqual(acknowledgement, { + operationId: 'operation-detach-in-flight', + outcome: 'accepted', + detail: 'detached', + }) + assert.deepEqual(await streamCompletion, { done: true, value: undefined }) + assert.equal(starts, 1) + assert.equal( + ( + await execution.detachRun({ + operationId: 'operation-detach-in-flight-retry', + runId: runInput.runId, + }) + ).outcome, + 'already-applied', + ) +}) + test('ambiguous cancellation effects reconcile or remain unknown', async () => { for (const effect of ['unknown', 'not_live'] as const) { const exact = controlRef(`ambiguous-${effect}`) diff --git a/test/support/tangle-retained-sandbox.ts b/test/support/tangle-retained-sandbox.ts new file mode 100644 index 0000000..876ad26 --- /dev/null +++ b/test/support/tangle-retained-sandbox.ts @@ -0,0 +1,381 @@ +import type { + AgentRunCancellationAcknowledgement, + AgentRunCancellationRequest, + AgentExactRunControlRef, + AgentProfile, +} from '@tangle-network/agent-interface' +import { + createTangleProvider, + defaultTangleSandboxCapabilities, + type SandboxClientLike, + type SandboxInstanceLike, +} from '@tangle-network/agent-provider-tangle' +import type { CreateSandboxOptions, SandboxEvent } from '@tangle-network/sandbox' +import type { PreparedTangleRetainedConnection } from '../../src/adapters/runtime/production-tangle-sandbox-backend.js' +import { safeExecutionId } from '../../src/adapters/runtime/production-backend-common.js' +import { observeSandboxClient } from '../../src/adapters/runtime/sandbox-observation.js' +import { + retainedSandboxIdentity, + retainedSandboxLifecycle, + withRetainedSandboxPolicy, +} from '../../src/adapters/runtime/tangle-sandbox-retention.js' + +interface FakeExecution { + readonly sessionId: string + readonly executionId: string + readonly controlRef: AgentExactRunControlRef + readonly events: SandboxEvent[] + status: 'running' | 'completed' | 'cancelled' | 'failed' + text: string + error?: string + readonly waiters: Set<() => void> +} + +export interface FakeRetainedBox { + readonly id: string + readonly idempotencyKey: string + readonly name?: string + readonly metadata?: Record + deleted: boolean +} + +/** Stateful double for the exact Tangle SDK surface used by provider 0.6.3. */ +export class FakeTangleRetainedSandbox { + readonly createCalls: CreateSandboxOptions[] = [] + readonly dispatches: Array<{ + readonly boxId: string + readonly sessionId: string + readonly executionId: string + readonly prompt: string + }> = [] + readonly cancellations: AgentRunCancellationRequest[] = [] + failDispatch = false + failDelete = false + providerRunId?: string + + readonly #boxesByKey = new Map() + readonly #boxesById = new Map() + readonly #executions = new Map() + readonly #cancellationDigests = new Map() + + get boxes(): readonly FakeRetainedBox[] { + return [...this.#boxesById.values()].filter((box) => !box.deleted) + } + + client(): SandboxClientLike { + return { + create: async (options) => { + const create = structuredClone(options ?? {}) + this.createCalls.push(create) + const key = create.idempotencyKey + if (typeof key !== 'string' || key.length === 0) { + throw new Error('Fake Tangle create requires idempotencyKey') + } + const existing = this.#boxesByKey.get(key) + const box = existing ?? this.#createBox(key, create) + return this.#instance(box) + }, + get: async (id) => { + const box = this.#boxesById.get(id) + return box === undefined || box.deleted ? null : this.#instance(box) + }, + list: async () => this.boxes.map((box) => this.#instance(box)), + describePlacement: () => ({ + kind: 'sandbox', + machineId: 'machine-fake-retained', + region: 'test-region', + }), + } + } + + complete(executionId: string, text: string): void { + this.#settle(executionId, 'completed', text) + } + + controlRefForExecution(executionId: string): AgentExactRunControlRef | null { + const exact = this.#executions.get(executionId)?.controlRef + return exact === undefined ? null : structuredClone(exact) + } + + #createBox(key: string, options: CreateSandboxOptions): FakeRetainedBox { + const box: FakeRetainedBox = { + id: `sandbox-${key}`, + idempotencyKey: key, + ...(options.name === undefined ? {} : { name: options.name }), + ...(options.metadata === undefined ? {} : { metadata: structuredClone(options.metadata) }), + deleted: false, + } + this.#boxesByKey.set(key, box) + this.#boxesById.set(box.id, box) + return box + } + + #instance(box: FakeRetainedBox): SandboxInstanceLike { + const sandbox = this + return { + id: box.id, + ...(box.name === undefined ? {} : { name: box.name }), + ...(box.metadata === undefined ? {} : { metadata: box.metadata }), + status: box.deleted ? 'stopped' : 'running', + async refresh() {}, + async delete() { + if (sandbox.failDelete) throw new Error('Injected Tangle delete failure') + box.deleted = true + }, + async dispatchPrompt(message, options) { + if (sandbox.failDispatch) throw new Error('Injected Tangle dispatch failure') + const sessionId = options?.sessionId + const executionId = options?.executionId + const controlRef = options?.runControlRef + if ( + typeof sessionId !== 'string' || + typeof executionId !== 'string' || + controlRef === undefined + ) { + throw new Error('Fake Tangle dispatch requires exact control coordinates') + } + const exactControlRef = + sandbox.providerRunId === undefined + ? controlRef + : Object.freeze({ ...controlRef, runId: sandbox.providerRunId }) + const existing = sandbox.#executions.get(executionId) + if (existing !== undefined) { + return { + sessionId, + executionId, + dispatched: false, + alreadyExisted: true, + runControlRef: existing.controlRef, + } + } + const execution: FakeExecution = { + sessionId, + executionId, + controlRef: exactControlRef, + events: [ + sandbox.#event(executionId, sessionId, 1, 'started'), + sandbox.#event(executionId, sessionId, 2, 'processing'), + ], + status: 'running', + text: '', + waiters: new Set(), + } + sandbox.#executions.set(executionId, execution) + sandbox.dispatches.push({ + boxId: box.id, + sessionId, + executionId, + prompt: typeof message === 'string' ? message : JSON.stringify(message), + }) + return { + sessionId, + executionId, + dispatched: true, + alreadyExisted: false, + status: 'running', + runControlRef: exactControlRef, + } + }, + async *streamPrompt(_message, options) { + const executionId = options?.executionId + if (typeof executionId !== 'string') return + const execution = sandbox.#requireExecution(executionId) + let index = 0 + if (options?.lastEventId && options.lastEventId !== '0') { + const cursorIndex = execution.events.findIndex( + (event) => event.id === options.lastEventId, + ) + if (cursorIndex < 0) throw new Error('Fake Tangle cursor does not exist') + index = cursorIndex + 1 + } + while (true) { + options?.signal?.throwIfAborted() + while (index < execution.events.length) { + const event = execution.events[index] + index += 1 + if (event !== undefined) yield event + } + if (execution.status !== 'running') return + await sandbox.#wait(execution, options?.signal) + } + }, + session(sessionId) { + return { + id: sessionId, + async status() { + const execution = [...sandbox.#executions.values()] + .filter((candidate) => candidate.sessionId === sessionId) + .at(-1) + return { status: execution?.status ?? 'running' } + }, + async *events() {}, + async result(options) { + const executionId = options?.executionId + if (typeof executionId !== 'string') { + throw new Error('Fake Tangle result requires executionId') + } + const execution = sandbox.#requireExecution(executionId) + while (execution.status === 'running') { + await sandbox.#wait(execution, options?.signal) + } + const success = execution.status === 'completed' + return { + executionId, + success, + status: success ? 'success' : 'failed', + durationMs: 5, + response: execution.text, + ...(execution.error === undefined ? {} : { error: execution.error }), + } + }, + async prompt() { + throw new Error('Fake Tangle prompt is not used by retained dispatch') + }, + async interrupt(options) { + const executionId = options?.executionId + if (typeof executionId !== 'string') return { cancelled: false } + const execution = sandbox.#executions.get(executionId) + if (execution === undefined || execution.status !== 'running') { + return { cancelled: false } + } + sandbox.#settle(executionId, 'cancelled', '') + return { cancelled: true } + }, + async cancelRun( + request: AgentRunCancellationRequest, + ): Promise { + const prior = sandbox.#cancellationDigests.get(request.operationId) + if (prior !== undefined && prior !== request.requestDigest) { + return { ...request, status: 'conflict', effect: 'unknown' } + } + sandbox.#cancellationDigests.set(request.operationId, request.requestDigest) + sandbox.cancellations.push(structuredClone(request)) + const execution = sandbox.#requireExecution(request.run.executionId) + if (execution.status === 'running') { + sandbox.#settle(execution.executionId, 'cancelled', '') + } + return { + operationId: request.operationId, + requestDigest: request.requestDigest, + run: request.run, + status: prior === undefined ? 'accepted' : 'replayed', + effect: 'cancelled', + } + }, + } + }, + } + } + + #event( + executionId: string, + sessionId: string, + sequence: number, + status: 'started' | 'processing', + ): SandboxEvent { + return { + type: 'status', + id: `event-${executionId}-${sequence}`, + data: { executionId, sessionId, status, normalized: { type: 'status', status } }, + } as SandboxEvent + } + + #settle(executionId: string, status: FakeExecution['status'], text: string): void { + const execution = this.#requireExecution(executionId) + execution.status = status + execution.text = text + for (const waiter of execution.waiters) waiter() + execution.waiters.clear() + } + + #requireExecution(executionId: string): FakeExecution { + const execution = this.#executions.get(executionId) + if (execution === undefined) throw new Error(`Unknown fake execution ${executionId}`) + return execution + } + + async #wait(execution: FakeExecution, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + await new Promise((resolve, reject) => { + const finish = () => { + signal?.removeEventListener('abort', abort) + resolve() + } + const abort = () => { + execution.waiters.delete(finish) + reject(signal?.reason ?? new DOMException('Aborted', 'AbortError')) + } + execution.waiters.add(finish) + signal?.addEventListener('abort', abort, { once: true }) + }) + } +} + +/** Test-only provider proof. Production must obtain these claims from the published provider. */ +export async function prepareFakeTangleRetainedConnection(input: { + readonly sandbox: FakeTangleRetainedSandbox + readonly profile: Readonly + readonly runId: string + readonly providerSessionId?: string + readonly idleTtlSeconds?: number +}): Promise { + const runner = 'opencode' as const + const model = input.profile.model?.default + if (typeof model !== 'string') throw new Error('Fake retained connection requires one model') + const providerSessionId = + input.providerSessionId ?? `session-braid-${safeExecutionId(input.runId)}` + const identity = retainedSandboxIdentity(providerSessionId) + const idleTtlSeconds = input.idleTtlSeconds ?? 1_800 + const base = defaultTangleSandboxCapabilities(runner) + const declared = { + ...base, + sessions: { ...base.sessions, continue: true }, + retainedControl: { + exactRunIdentity: true, + resultIdentity: true, + eventIdentity: true, + cancellationIdempotency: true, + }, + } + const lifecycle = retainedSandboxLifecycle(idleTtlSeconds) + const observed = observeSandboxClient( + withRetainedSandboxPolicy(input.sandbox.client(), idleTtlSeconds), + lifecycle, + ) + const provider = createTangleProvider({ + client: observed.client, + defaultBackend: runner, + name: 'tangle-sandbox', + capabilities: declared, + }) + const reported = await provider.capabilities() + const capabilities = Object.freeze({ + ...reported, + sessions: Object.freeze({ ...reported.sessions, continue: false }), + }) + return Object.freeze({ + profile: input.profile, + model, + runner, + provider, + capabilities, + observation: observed.observation, + providerSessionId, + environmentIdempotencyKey: identity.environmentIdempotencyKey, + environmentName: identity.name, + environmentMetadata: identity.metadata, + idleTtlSeconds, + discoverControlRef: async (braidRunId: string) => + input.sandbox.controlRefForExecution(safeExecutionId(braidRunId)), + materializationReceipt: Object.freeze({ + provider: 'tangle-sandbox', + backend: 'environment-provider', + lifecycle: 'retained', + cleanup: 'explicit', + continuity: 'unavailable', + portableContext: 'unavailable', + model, + runner, + }), + }) +} diff --git a/test/tangle-retained-lifecycle.test.ts b/test/tangle-retained-lifecycle.test.ts new file mode 100644 index 0000000..a37f4e9 --- /dev/null +++ b/test/tangle-retained-lifecycle.test.ts @@ -0,0 +1,323 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { defineAgentProfile } from '@tangle-network/agent-interface' +import { + resolveTangleSandboxBackend, + resolveTangleSandboxRetainedConnection, +} from '../src/adapters/runtime/production-tangle-sandbox-backend.js' +import { capabilitiesForConnection } from '../src/adapters/connections/production-connection-providers.js' +import { + createTangleRetainedPlan, + startTangleRetainedRun, +} from '../src/adapters/runtime/tangle-retained-run.js' +import { withRetainedSandboxPolicy } from '../src/adapters/runtime/tangle-sandbox-retention.js' +import { ConnectionRegistry } from '../src/app/connections.js' +import type { ObservableSandboxClient } from '../src/adapters/runtime/sandbox-observation-types.js' +import type { ConnectionRecord, ConnectionTransportOptions } from '../src/domain/entities.js' +import { createConnectionId } from '../src/domain/ids.js' +import { assertConnectionRecord } from '../src/domain/invariants-profile.js' +import { + FakeTangleRetainedSandbox, + prepareFakeTangleRetainedConnection, +} from './support/tangle-retained-sandbox.js' + +const now = '2026-08-12T12:00:00.000Z' +const profile = defineAgentProfile({ + name: 'Retained Tangle test', + harness: 'opencode', + model: { provider: 'openai', default: 'openai/gpt-5' }, +}) + +function record(providerOptions: ConnectionTransportOptions): ConnectionRecord { + return { + id: createConnectionId('connection-tangle-retained-test'), + kind: 'tangle-sandbox', + name: 'Tangle Sandbox', + endpoint: 'https://sandbox.test', + providerOptions, + createdAt: now, + updatedAt: now, + lastHealth: { status: 'unknown' }, + } +} + +function setup(sandbox: FakeTangleRetainedSandbox, providerSessionId?: string) { + const connection = record({ + transport: 'https', + lifecycle: 'retained', + idleTtlSeconds: 1_800, + }) + const options = { + connections: new ConnectionRegistry([connection]), + select: () => ({ connection: { connectionId: connection.id } }), + sandboxClient: sandbox.client(), + } + const input = { + operationId: 'operation-tangle-retained', + runId: 'run/tangle-retained', + text: 'Prove retained cloud execution.', + profile, + connectionId: connection.id, + signal: new AbortController().signal, + ...(providerSessionId === undefined ? {} : { sessionId: providerSessionId }), + } + const selection = { connection: { connectionId: connection.id } } + return { connection, options, input, selection } +} + +test('retained lifecycle configuration is explicit and bounded', () => { + assert.doesNotThrow(() => + assertConnectionRecord(record({ lifecycle: 'retained', idleTtlSeconds: 1_800 })), + ) + assert.throws( + () => assertConnectionRecord(record({ lifecycle: 'retained' })), + /requires idleTtlSeconds/u, + ) + assert.throws( + () => assertConnectionRecord(record({ lifecycle: 'retained', idleTtlSeconds: 59 })), + /60 to 604800/u, + ) + assert.throws( + () => assertConnectionRecord(record({ lifecycle: 'ephemeral', idleTtlSeconds: 1_800 })), + /requires lifecycle=retained/u, + ) +}) + +test('retained policy fails closed for an ambiguous completed-turn 404', async () => { + let responseStatus = 404 + let dispatches = 0 + const box = { + id: 'sandbox-completed-turn-contract', + async *streamPrompt() {}, + async findCompletedTurn() { + throw Object.assign(new Error(`${responseStatus} completed-turn response`), { + status: responseStatus, + code: responseStatus === 404 ? 'NOT_FOUND' : 'SERVER_ERROR', + }) + }, + async dispatchPrompt( + this: { + findCompletedTurn( + turnId: string, + options: { readonly sessionId: string }, + ): Promise + }, + _message: unknown, + options: { readonly sessionId?: string; readonly turnId?: string } = {}, + ) { + const sessionId = options.sessionId ?? 'missing-session' + const turnId = options.turnId ?? 'missing-turn' + const cached = await this.findCompletedTurn(turnId, { sessionId }) + dispatches += 1 + return { sessionId, dispatched: true, cached } + }, + } + const client = withRetainedSandboxPolicy({ create: async () => box as never }, 300) + const retained = await client.create() + const dispatch = retained.dispatchPrompt + assert.ok(dispatch) + + await assert.rejects( + dispatch.call(retained, 'run once', { + sessionId: 'session-completed-turn-contract', + turnId: 'turn-completed-turn-contract', + }), + /404 completed-turn response/u, + ) + assert.equal(dispatches, 0) + + responseStatus = 500 + await assert.rejects( + ( + retained as typeof retained & { + findCompletedTurn(turnId: string, options: { sessionId: string }): Promise + } + ).findCompletedTurn('turn-completed-turn-contract', { + sessionId: 'session-completed-turn-contract', + }), + /500 completed-turn response/u, + ) +}) + +test('retained policy preserves account observation methods from the Sandbox client', async () => { + const identity = { + customerId: 'customer-retained-policy', + billingOwnerId: 'billing-retained-policy', + apiKeyId: null, + billingDelegationAuthorized: false, + } + const usage = { + computeMinutes: 1, + gpuSeconds: 0, + gpuCostUsd: 0, + gpuProviderCostUsd: 0, + activeSandboxes: 0, + totalSandboxes: 1, + periodStart: new Date('2026-08-12T00:00:00.000Z'), + periodEnd: new Date('2026-08-13T00:00:00.000Z'), + } + const source = { + async create() { + return {} as never + }, + async getIdentity() { + return identity + }, + async usage() { + return usage + }, + } as ObservableSandboxClient + const retained = withRetainedSandboxPolicy(source, 300) as ObservableSandboxClient + + assert.deepEqual(await retained.getIdentity?.(), identity) + assert.deepEqual(await retained.usage?.(), usage) +}) + +test('retained capability and resolution fail closed without provider-backed lookup', async () => { + const sandbox = new FakeTangleRetainedSandbox() + const { connection, options, input, selection } = setup(sandbox) + const report = await capabilitiesForConnection(connection, options) + assert.equal(report.runtime.streaming.live, false) + assert.equal(report.actions.stream, false) + await assert.rejects( + resolveTangleSandboxRetainedConnection(options, input, selection, connection.id), + /requires provider-backed lookup/u, + ) + + assert.equal(sandbox.createCalls.length, 0) + await assert.rejects( + resolveTangleSandboxBackend(options, input, selection, connection.id), + /retained execution port/u, + ) +}) + +test('retained resolution still requires exact control after lookup is configured', async () => { + const sandbox = new FakeTangleRetainedSandbox() + const { connection, options, input, selection } = setup(sandbox) + await assert.rejects( + resolveTangleSandboxRetainedConnection( + { ...options, tangleRetainedControlLookup: async () => null }, + input, + selection, + connection.id, + ), + /did not report exact retained-run control/u, + ) + assert.equal(sandbox.createCalls.length, 0) +}) + +test('one retained plan uses exact tags, bounded idle expiry, replay, and result identity', async () => { + const sandbox = new FakeTangleRetainedSandbox() + const { input } = setup(sandbox) + const prepared = await prepareFakeTangleRetainedConnection({ + sandbox, + profile, + runId: input.runId, + }) + const plan = createTangleRetainedPlan(prepared, input.runId) + assert.equal(plan.capabilities.sessions.continue, false) + assert.equal(plan.capabilities.controls.status, false) + const handle = await startTangleRetainedRun(plan, input) + + assert.equal(sandbox.createCalls.length, 1) + assert.equal(sandbox.createCalls[0]?.idempotencyKey, prepared.environmentIdempotencyKey) + assert.equal(sandbox.createCalls[0]?.name, prepared.environmentName) + assert.equal(sandbox.createCalls[0]?.idleTimeoutSeconds, 1_800) + assert.equal(sandbox.createCalls[0]?.ephemeral, false) + assert.deepEqual(sandbox.createCalls[0]?.metadata, prepared.environmentMetadata) + assert.equal(handle.controlRef.environmentId, sandbox.boxes[0]?.id) + assert.equal(handle.controlRef.sessionId, prepared.providerSessionId) + assert.equal(handle.controlRef.executionId, 'run-tangle-retained') + assert.match(handle.controlRef.requestDigest, /^sha256:[0-9a-f]{64}$/u) + + sandbox.complete(handle.controlRef.executionId, 'RETAINED_OK') + const events = [] + for await (const event of handle.events()) events.push(event) + assert.deepEqual( + events.map((event) => event.cursor), + [`event-${handle.controlRef.executionId}-1`, `event-${handle.controlRef.executionId}-2`], + ) + const result = await handle.result() + assert.equal(result.text, 'RETAINED_OK') + assert.equal(result.metadata?.executionId, handle.controlRef.executionId) + + const reconnected = await plan.reconnect(handle.controlRef) + assert.deepEqual(reconnected?.controlRef, handle.controlRef) + const replayed = [] + const firstEvent = events[0] + if (firstEvent?.cursor === undefined) throw new Error('First retained event omitted its cursor') + for await (const event of reconnected?.events({ + after: { cursor: firstEvent.cursor, sequence: firstEvent.sequence }, + }) ?? []) { + replayed.push(event) + } + assert.deepEqual( + replayed.map((event) => event.cursor), + [events[1]?.cursor], + ) +}) + +test('ambiguous dispatch failure never deletes the retained environment', async () => { + const sandbox = new FakeTangleRetainedSandbox() + sandbox.failDispatch = true + const { input } = setup(sandbox) + const prepared = await prepareFakeTangleRetainedConnection({ + sandbox, + profile, + runId: input.runId, + }) + const plan = createTangleRetainedPlan(prepared, input.runId) + + await assert.rejects(startTangleRetainedRun(plan, input), /dispatch failure/u) + assert.equal(sandbox.createCalls.length, 1) + assert.equal(sandbox.boxes.length, 1) + assert.equal(sandbox.createCalls[0]?.idleTimeoutSeconds, 1_800) +}) + +test('a failed retry never deletes a pre-existing retained workspace', async () => { + const sandbox = new FakeTangleRetainedSandbox() + const { input } = setup(sandbox, 'session-braid-existing-workspace') + const providerSessionId = input.sessionId + assert.ok(providerSessionId) + const prepared = await prepareFakeTangleRetainedConnection({ + sandbox, + profile, + runId: input.runId, + providerSessionId, + }) + const first = await startTangleRetainedRun(createTangleRetainedPlan(prepared, input.runId), input) + sandbox.complete(first.controlRef.executionId, 'EXISTING_WORKSPACE') + sandbox.failDispatch = true + const retryInput = { + ...input, + operationId: 'operation-tangle-retained-retry', + runId: 'run/tangle-retained-retry', + } + + await assert.rejects( + startTangleRetainedRun(createTangleRetainedPlan(prepared, retryInput.runId), retryInput), + /dispatch failure/u, + ) + assert.equal(sandbox.boxes.length, 1) +}) + +test('exact cancellation is retry-safe through the Runtime handle', async () => { + const sandbox = new FakeTangleRetainedSandbox() + const { input } = setup(sandbox) + const prepared = await prepareFakeTangleRetainedConnection({ + sandbox, + profile, + runId: input.runId, + }) + const plan = createTangleRetainedPlan(prepared, input.runId) + const handle = await startTangleRetainedRun(plan, input) + const first = await handle.cancel({ operationId: 'operation-cancel', reason: 'test' }) + const replay = await handle.cancel({ operationId: 'operation-cancel', reason: 'test' }) + + assert.equal(first.status, 'accepted') + assert.equal(first.effect, 'cancelled') + assert.equal(replay.status, 'replayed') + assert.equal(replay.effect, 'cancelled') + assert.equal(sandbox.cancellations.length, 2) + assert.equal(sandbox.cancellations[0]?.run.executionId, handle.controlRef.executionId) +}) diff --git a/test/tangle-retained-restart.test.ts b/test/tangle-retained-restart.test.ts new file mode 100644 index 0000000..047ef22 --- /dev/null +++ b/test/tangle-retained-restart.test.ts @@ -0,0 +1,205 @@ +import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { defineAgentProfile } from '@tangle-network/agent-interface' +import { MemoryCredentialStore } from '../src/adapters/credentials/memory.js' +import { TangleRetainedExecutionPort } from '../src/adapters/runtime/tangle-retained-execution.js' +import { + createDurableBraidApplication, + type DurableBraidApplication, +} from '../src/app/composition.js' +import { isRuntimeEventEnvelope } from '../src/domain/runtime-events.js' +import { RandomIds } from '../src/ports/ids.js' +import { + FakeTangleRetainedSandbox, + prepareFakeTangleRetainedConnection, +} from './support/tangle-retained-sandbox.js' + +const idleTtlSeconds = 1_800 +const profile = defineAgentProfile({ + name: 'Retained sandbox proof', + harness: 'opencode', + model: { provider: 'openai', default: 'openai/gpt-5' }, +}) + +function retainedExecution(sandbox: FakeTangleRetainedSandbox): TangleRetainedExecutionPort { + const prepare = async (input: { + readonly runId: string + readonly providerSessionId?: string + }) => { + const prepared = await prepareFakeTangleRetainedConnection({ + sandbox, + profile, + runId: input.runId, + idleTtlSeconds, + ...(input.providerSessionId === undefined + ? {} + : { providerSessionId: input.providerSessionId }), + }) + return prepared + } + return new TangleRetainedExecutionPort({ + resolve: (input) => + prepare({ + runId: input.runId, + ...(input.sessionId === undefined ? {} : { providerSessionId: input.sessionId }), + }), + recover: prepare, + }) +} + +async function waitFor(predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs + while (!predicate()) { + if (Date.now() >= deadline) throw new Error('Timed out waiting for retained Tangle state') + await new Promise((resolve) => setTimeout(resolve, 2)) + } +} + +test('one retained cloud run survives a Braid restart without claiming native continuation', async () => { + const root = await mkdtemp(join(tmpdir(), 'braid-tangle-retained-')) + const sandbox = new FakeTangleRetainedSandbox() + const path = join(root, 'braid.db') + const credentials = new MemoryCredentialStore() + let first: DurableBraidApplication | undefined + let restarted: DurableBraidApplication | undefined + try { + first = await createDurableBraidApplication({ + path, + workspaceRoot: root, + credentialStore: credentials, + profile, + execution: retainedExecution(sandbox), + ids: new RandomIds(), + }) + first.app.initialize(root) + + const turn = first.app.send({ + operationId: 'operation-tangle-retained-turn', + text: 'Keep working while Braid restarts.', + }) + await turn.admissionReady + await waitFor(() => sandbox.dispatches.length === 1) + await waitFor(() => { + const run = first?.app.state().runs.find((candidate) => candidate.id === turn.runId) + return run?.controlRef !== undefined && (run.lastProviderSequence ?? 0) >= 2 + }) + const firstRun = first.app.state().runs.find((candidate) => candidate.id === turn.runId) + assert.equal(firstRun?.capabilities.sessions.continue, false) + assert.equal(firstRun?.capabilities.controls.status, false) + assert.equal(firstRun?.capabilities.streaming.replay, true) + assert.equal(firstRun?.capabilities.streaming.detach, true) + assert.equal(sandbox.createCalls[0]?.idleTimeoutSeconds, idleTtlSeconds) + assert.equal(sandbox.boxes.length, 1) + const beforeRestart = first.app.state().runs.find((candidate) => candidate.id === turn.runId) + assert.equal(beforeRestart?.controlRef?.environmentId, sandbox.boxes[0]?.id) + const detached = await first.app.detachRun({ + operationId: 'operation-tangle-retained-detach', + runId: turn.runId, + }) + await detached.completion + await turn.completion + assert.equal( + first.app.state().runs.find((candidate) => candidate.id === turn.runId)?.status, + 'detached', + ) + await first.app.close() + first = undefined + + sandbox.complete(sandbox.dispatches[0]?.executionId ?? '', 'SURVIVED_RESTART') + restarted = await createDurableBraidApplication({ + path, + workspaceRoot: root, + credentialStore: credentials, + profile, + execution: retainedExecution(sandbox), + ids: new RandomIds(), + }) + const restored = restarted.app.state().runs.find((candidate) => candidate.id === turn.runId) + assert.equal(restored?.status, 'detached') + assert.deepEqual(restored?.controlRef, beforeRestart?.controlRef) + + await restarted.app.reconnectRun({ + operationId: 'operation-tangle-retained-reconnect', + runId: turn.runId, + }) + const completed = restarted.app.state() + assert.equal( + completed.runs.find((candidate) => candidate.id === turn.runId)?.status, + 'completed', + ) + assert.equal(completed.messages.at(-1)?.text, 'SURVIVED_RESTART') + assert.deepEqual(completed.missingHistory, []) + assert.equal( + restarted.app + .events() + .filter((entry) => entry.event.kind === 'run.finished' && entry.event.runId === turn.runId) + .length, + 1, + ) + assert.equal(sandbox.dispatches.length, 1) + } finally { + await first?.app.close().catch(() => undefined) + await restarted?.app.close().catch(() => undefined) + await rm(root, { recursive: true, force: true }) + } +}) + +test('provider lookup recovers the pre-journal crash window without a saved reference', async () => { + const root = await mkdtemp(join(tmpdir(), 'braid-tangle-crash-window-')) + const sandbox = new FakeTangleRetainedSandbox() + const runId = 'run-tangle-crash-window' + const providerRunId = 'provider-run-tangle-crash-window' + sandbox.providerRunId = providerRunId + const providerSessionId = 'session-braid-existing-conversation' + const abort = new AbortController() + try { + const first = retainedExecution(sandbox) + if (first.admit === undefined) throw new Error('Retained admission is unavailable') + const input = { + operationId: 'operation-tangle-crash-window', + runId, + text: 'Continue the existing retained conversation.', + profile, + workspaceRoot: root, + sessionId: providerSessionId, + signal: abort.signal, + } + const admission = await first.admit(input) + assert.equal(admission.providerSessionId, providerSessionId) + assert.equal(sandbox.boxes.length, 0) + + const stream = first.streamTurn(input)[Symbol.asyncIterator]() + const observed = await stream.next() + assert.equal(observed.done, false) + const event = + observed.value !== undefined && isRuntimeEventEnvelope(observed.value) + ? observed.value.event + : undefined + assert.equal(event?.type, 'braid.execution.observed') + const controlRef = event?.type === 'braid.execution.observed' ? event.controlRef : undefined + assert.equal(controlRef?.sessionId, providerSessionId) + assert.equal(controlRef?.runId, providerRunId) + await stream.return?.(undefined) + + const restarted = retainedExecution(sandbox) + if (restarted.status === undefined || restarted.cancelRun === undefined) { + throw new Error('Retained recovery controls are unavailable') + } + assert.equal(await restarted.status({ runId, providerSessionId }), null) + assert.equal(await restarted.status({ runId, providerSessionId, controlRef }), null) + const cancelled = await restarted.cancelRun({ + operationId: 'operation-tangle-crash-window-cancel', + runId, + providerSessionId, + }) + assert.equal(cancelled.outcome, 'accepted') + assert.equal(sandbox.cancellations.length, 1) + assert.equal(sandbox.cancellations[0]?.run.runId, providerRunId) + } finally { + abort.abort() + await rm(root, { recursive: true, force: true }) + } +}) diff --git a/test/tangle-sandbox-braid-soak-script.test.mjs b/test/tangle-sandbox-braid-soak-script.test.mjs index a9ec8d1..372dc6e 100644 --- a/test/tangle-sandbox-braid-soak-script.test.mjs +++ b/test/tangle-sandbox-braid-soak-script.test.mjs @@ -3,69 +3,357 @@ import test from 'node:test' import { setTimeout as delay } from 'node:timers/promises' import { runBraidSandboxSoak } from '../scripts/live-required/tangle-sandbox-braid-soak.mjs' +function mergeRecords(base, override) { + if ( + base !== null && + typeof base === 'object' && + !Array.isArray(base) && + override !== null && + typeof override === 'object' && + !Array.isArray(override) + ) { + return Object.fromEntries( + [...new Set([...Object.keys(base), ...Object.keys(override)])].map((key) => [ + key, + mergeRecords(base[key], override[key]), + ]), + ) + } + return override === undefined ? base : override +} + function passedProof(index, overrides = {}) { - return { - status: 'passed', - proofId: `proof-${index}`, - processes: { binarySha256: 'b'.repeat(64) }, - progress: { - firstControlRef: { - provider: 'tangle-sandbox', - environmentId: `sandbox-${index}`, - sessionId: `session-${index}`, - executionId: `execution-${index}`, - runId: `provider-run-${index}`, - requestDigest: `sha256:${'a'.repeat(64)}`, - }, - }, - cleanup: { - exactResource: true, - activeResourceDelta: 0, - identity: { confirmed: true, remainingIds: [] }, + const accountIdentityDigest = 'd'.repeat(64) + const environmentId = `sandbox-${index}` + const localEnvironmentId = `local-environment-${index}` + const sessionId = `session-${index}` + const firstRunId = `local-run-${index}` + const followUpRunId = `follow-up-run-${index}` + const cancelRunId = `cancel-run-${index}` + const firstControlRef = { + provider: 'tangle-sandbox', + environmentId, + sessionId, + executionId: `execution-${index}`, + runId: `provider-run-${index}`, + requestDigest: `sha256:${'a'.repeat(64)}`, + } + const followUpControlRef = { + ...firstControlRef, + executionId: `follow-up-execution-${index}`, + runId: `follow-up-provider-run-${index}`, + requestDigest: `sha256:${'b'.repeat(64)}`, + } + const cancelControlRef = { + ...firstControlRef, + executionId: `cancel-execution-${index}`, + runId: `cancel-provider-run-${index}`, + requestDigest: `sha256:${'c'.repeat(64)}`, + } + const snapshot = ({ id, operationId, status, controlRef, cursor }) => ({ + id, + operationId, + status, + environmentId: localEnvironmentId, + providerSessionId: sessionId, + cursor, + controlRef, + observations: { + localEnvironmentId, + providerEnvironmentId: environmentId, + environmentRecord: { id: localEnvironmentId, providerEnvironmentId: environmentId }, + run: { id, status }, + environment: { id: localEnvironmentId, providerEnvironmentId: environmentId }, }, - accountIdentityConsistency: { - stable: true, - customerId: 'customer-1', - billingOwnerId: 'billing-owner-1', + }) + const beforeUsage = { + activeSandboxes: 0, + totalSandboxes: index, + computeMinutes: index, + gpuSeconds: 0, + gpuCostUsd: 0, + } + const afterUsage = { + activeSandboxes: 0, + totalSandboxes: index + 1, + computeMinutes: index + 1, + gpuSeconds: 0, + gpuCostUsd: 0, + } + const verifiedProcessCleanup = { + exited: true, + descendantsVerified: true, + exit: { code: 0, signal: null }, + } + const telemetryFields = { + tokens: { status: 'observed', input: 10, output: 2 }, + cost: { status: 'observed', usd: 0.001 }, + endToEndDuration: { status: 'observed', milliseconds: 100 }, + model: { status: 'observed', value: 'test-model' }, + environment: { status: 'observed', value: environmentId }, + runtimeEndpoint: { status: 'observed', host: 'sandbox.test' }, + machine: { status: 'unavailable' }, + region: { status: 'observed', value: 'test-region' }, + requestedResources: { status: 'provider-default' }, + resourceSample: { status: 'observed', value: { activeSandboxes: 0 } }, + account: { + status: 'observed', + value: { identityDigest: accountIdentityDigest }, }, - usage: [ - { - phase: 'before', - status: 'observed', - value: { activeSandboxes: 0, totalSandboxes: index, computeMinutes: index }, + } + const telemetryDisclosure = { + completeDisclosure: true, + unavailable: ['machine-id:not-supported'], + fields: telemetryFields, + } + return mergeRecords( + { + status: 'passed', + proofId: `proof-${index}`, + resourceIdentity: { + observed: true, + id: environmentId, + name: `braid-${sessionId}`, + metadata: { + owner: 'braid', + lifecycle: 'retained', + providerSessionId: sessionId, + }, }, - { - phase: 'after', - status: 'observed', - value: { activeSandboxes: 0, totalSandboxes: index + 1, computeMinutes: index + 1 }, + processes: { + first: { signal: 'SIGKILL', code: null, sent: true, cleanup: verifiedProcessCleanup }, + cancelled: { cleanup: verifiedProcessCleanup }, + retry: { cleanup: verifiedProcessCleanup }, + localRunCountAfterReconnect: 1, + binarySha256: 'b'.repeat(64), }, - ], - spend: { - rows: [ + runs: { + first: snapshot({ + id: firstRunId, + operationId: `op-${index}-first`, + status: 'running', + controlRef: firstControlRef, + cursor: `cursor-first-${index}`, + }), + resumed: snapshot({ + id: firstRunId, + operationId: `op-${index}-first`, + status: 'completed', + controlRef: firstControlRef, + cursor: `cursor-resumed-${index}`, + }), + followUp: snapshot({ + id: followUpRunId, + operationId: `op-${index}-follow-up`, + status: 'completed', + controlRef: followUpControlRef, + cursor: `cursor-follow-up-${index}`, + }), + cancelled: snapshot({ + id: cancelRunId, + operationId: `op-${index}-cancel`, + status: 'cancelled', + controlRef: cancelControlRef, + cursor: `cursor-cancel-${index}`, + }), + }, + progress: { + firstRunId, + cancelRunId, + providerEnvironmentId: environmentId, + firstControlRef, + resumeFromCursor: `cursor-first-${index}`, + freshControlRef: firstControlRef, + finalCursor: `cursor-resumed-${index}`, + }, + replay: { + firstVisibleEventCount: 2, + freshVisibleEventCount: 2, + freshVisibleEventIdsUnique: true, + resumeFromCursor: `cursor-first-${index}`, + finalCursor: `cursor-resumed-${index}`, + acknowledgedBeforeKillEventIds: [`event-first-${index}`], + freshVisibleEventIds: [`event-fresh-${index}`], + acknowledgedAndFreshIntersection: [], + progress: { acknowledgedSequence: 2, firstFreshSequence: 3 }, + reconnectRequest: { + command: 'reconnect', + operationId: `op-${index}-reconnect`, + params: { runId: firstRunId }, + }, + }, + cancellation: { + first: { type: 'ack', runId: cancelRunId, operationId: `op-${index}-cancel` }, + sameBody: { + type: 'ack', + runId: cancelRunId, + operationId: `op-${index}-cancel`, + replayed: true, + }, + changedBody: { type: 'error', code: 'OPERATION_CONFLICT' }, + remote: { + controlRef: cancelControlRef, + samples: [{ status: 'running' }, { status: 'cancelled' }], + settledStatus: 'cancelled', + messageCount: 1, + lateResult: false, + }, + }, + cleanup: { + exactResource: true, + mode: 'exact-owned-resource-set', + activeResourceDelta: 0, + activeResourceDeltaRequired: true, + usageObservationComplete: true, + usageDelta: { + activeSandboxes: 0, + totalSandboxes: 1, + computeMinutes: 1, + gpuSeconds: 0, + gpuCostUsd: 0, + unknownFields: [], + }, + identity: { + confirmed: true, + mode: 'exact-owned-resource-set', + matchedCount: 1, + removedIds: [environmentId], + deletions: [{ id: environmentId, confirmed: true }], + remainingIds: [], + }, + }, + accountIdentityConsistency: { + stable: true, + identityDigest: accountIdentityDigest, + }, + accountIdentities: [ + { + phase: 'before', + status: 'observed', + value: { identityDigest: accountIdentityDigest }, + }, { - label: 'resumed-first-turn', - tokens: { status: 'observed', input: 10 + index, output: 2 + index }, - cost: { status: 'observed', usd: 0.001 + index / 1_000 }, + phase: 'after', + status: 'observed', + value: { identityDigest: accountIdentityDigest }, }, + ], + account: { + identityDigest: accountIdentityDigest, + usage: afterUsage, + }, + usage: [ { - label: 'follow-up-turn', - tokens: { status: 'observed', input: 5, output: 1 }, - cost: { status: 'unavailable' }, + phase: 'before', + status: 'observed', + value: beforeUsage, }, { - label: 'cancelled-turn', - tokens: { status: 'unavailable' }, - cost: { status: 'unavailable' }, + phase: 'after', + status: 'observed', + value: afterUsage, }, ], + telemetry: { + ...telemetryDisclosure, + runs: { + first: telemetryDisclosure, + resumed: telemetryDisclosure, + followUp: telemetryDisclosure, + cancelled: telemetryDisclosure, + }, + }, + workspaceVerification: { + readMatched: true, + continuity: { matched: true }, + git: { exitCode: 0 }, + resourceSample: { status: 'observed', value: { activeSandboxes: 0 } }, + }, + followUpEvidence: { + visibleProviderEvents: 1, + continuity: { matched: true }, + }, + spend: { + rows: [ + { + label: 'resumed-first-turn', + tokens: { status: 'observed', input: 10 + index, output: 2 + index }, + cost: { status: 'observed', usd: 0.001 + index / 1_000 }, + }, + { + label: 'follow-up-turn', + tokens: { status: 'observed', input: 5, output: 1 }, + cost: { status: 'unavailable' }, + }, + { + label: 'cancelled-turn', + tokens: { status: 'unavailable' }, + cost: { status: 'unavailable' }, + }, + ], + }, + timing: { + totalMs: 100 + index, + workspace: { elapsedMs: 20 + index }, + reconnect: { elapsedMs: 30 + index }, + }, }, - timing: { - totalMs: 100 + index, - workspace: { elapsedMs: 20 + index }, - reconnect: { elapsedMs: 30 + index }, + overrides, + ) +} + +function proofWithUsage(index, beforeOverrides = {}, afterOverrides = {}) { + const base = passedProof(index) + const before = { ...base.usage[0].value, ...beforeOverrides } + const after = { ...base.usage[1].value, ...afterOverrides } + const delta = Object.fromEntries( + Object.keys(base.cleanup.usageDelta) + .filter((field) => field !== 'unknownFields') + .map((field) => [ + field, + typeof before[field] === 'number' && + Number.isFinite(before[field]) && + typeof after[field] === 'number' && + Number.isFinite(after[field]) + ? after[field] - before[field] + : null, + ]), + ) + delta.unknownFields = Object.keys(delta).filter((field) => delta[field] === null) + return mergeRecords(base, { + usage: [ + { phase: 'before', status: 'observed', value: before }, + { phase: 'after', status: 'observed', value: after }, + ], + cleanup: { + activeResourceDelta: delta.activeSandboxes, + usageDelta: delta, }, - ...overrides, + }) +} + +function proofWithCloudEnvironment(index, environmentId) { + const proof = structuredClone(passedProof(index)) + const rewriteControlRef = (controlRef) => { + controlRef.environmentId = environmentId + } + rewriteControlRef(proof.progress.firstControlRef) + rewriteControlRef(proof.progress.freshControlRef) + proof.progress.providerEnvironmentId = environmentId + for (const snapshot of Object.values(proof.runs)) { + rewriteControlRef(snapshot.controlRef) + snapshot.observations.providerEnvironmentId = environmentId + snapshot.observations.environmentRecord.providerEnvironmentId = environmentId + snapshot.observations.environment.providerEnvironmentId = environmentId + } + proof.resourceIdentity.id = environmentId + proof.cleanup.identity.removedIds = [environmentId] + proof.cancellation.remote.controlRef.environmentId = environmentId + proof.telemetry.fields.environment.value = environmentId + for (const disclosure of Object.values(proof.telemetry.runs)) { + disclosure.fields.environment.value = environmentId } + return proof } test('sandbox soak stops after a failed canary before spending on more runs', async () => { @@ -88,6 +376,24 @@ test('sandbox soak stops after a failed canary before spending on more runs', as assert.match(result.failures[0], /attempted 1 of 5/u) }) +test('sandbox soak keeps an unavailable cleanup count distinct from a measured leak', async () => { + const result = await runBraidSandboxSoak({ + runs: 1, + concurrency: 1, + stressRunner: async () => ({ + status: 'failed', + proofId: 'proof-unavailable-cleanup', + cleanup: { exactResource: false, activeResourceDelta: null }, + accountIdentityConsistency: null, + usage: [], + timing: { totalMs: 1 }, + }), + }) + + assert.equal(result.cleanup.exactResourcesRemaining, null) + assert.equal(result.cleanup.resourceProofsUnavailable, 1) +}) + test('sandbox soak preserves protected-provider unavailability', async () => { const unavailable = new Error('protected credential is unavailable') unavailable.unavailable = true @@ -133,6 +439,7 @@ test('sandbox soak runs one canary then a bounded parallel cohort with full summ assert.equal(result.latency.phases.workspace.n, 5) assert.equal(result.cleanup.exactProofs, 5) assert.equal(result.cleanup.exactResourcesRemaining, 0) + assert.equal(result.cleanup.resourceProofsUnavailable, 0) assert.deepEqual(result.sessionSpend.tokens, { observedRuns: 10, unavailableRuns: 5, @@ -145,23 +452,81 @@ test('sandbox soak runs one canary then a bounded parallel cohort with full summ assert.ok(Math.abs(result.sessionSpend.cost.usd - 0.015) < Number.EPSILON) }) +test('sandbox soak rejects a passed proof with missing durable retained fields', async () => { + const result = await runBraidSandboxSoak({ + runs: 1, + stressRunner: async () => + passedProof(0, { + progress: { freshControlRef: null }, + replay: { resumeFromCursor: '' }, + }), + }) + + assert.equal(result.status, 'failed') + assert.ok(result.failures.some((failure) => /freshControlRef.*incomplete/u.test(failure))) + assert.ok(result.failures.some((failure) => /replay\.resumeFromCursor/u.test(failure))) +}) + +test('sandbox soak rejects per-run active leaks hidden by an aggregate zero', async () => { + const result = await runBraidSandboxSoak({ + runs: 3, + concurrency: 2, + stressRunner: async ({ attemptIndex }) => { + if (attemptIndex === 0) return passedProof(0) + if (attemptIndex === 1) { + await delay(5) + return proofWithUsage(1, { activeSandboxes: 0 }, { activeSandboxes: 1 }) + } + await delay(15) + return proofWithUsage(2, { activeSandboxes: 1 }, { activeSandboxes: 0 }) + }, + }) + + assert.equal(result.status, 'failed') + assert.equal(result.cleanup.activeResourceDelta, 0) + assert.ok( + result.failures.some((failure) => + /run 2: proof account active-resource delta was 1/u.test(failure), + ), + ) + assert.ok( + result.failures.some((failure) => + /run 3: proof account active-resource delta was -1/u.test(failure), + ), + ) +}) + +test('sandbox soak rejects unknown account deltas instead of treating them as zero', async () => { + const result = await runBraidSandboxSoak({ + runs: 1, + stressRunner: async () => proofWithUsage(0, {}, { computeMinutes: null }), + }) + + assert.equal(result.status, 'failed') + assert.ok(result.failures.some((failure) => /computeMinutes.*unknown/u.test(failure))) + assert.ok(result.failures.some((failure) => /unknown fields.*computeMinutes/u.test(failure))) +}) + +test('sandbox soak rejects arbitrary cumulative account increases', async () => { + const result = await runBraidSandboxSoak({ + runs: 1, + stressRunner: async () => proofWithUsage(0, { totalSandboxes: 0 }, { totalSandboxes: 99 }), + }) + + assert.equal(result.status, 'failed') + assert.ok( + result.failures.some((failure) => + /totalSandboxes delta 99 exceeded retained-resource-creation bound 1/u.test(failure), + ), + ) +}) + test('sandbox soak rejects reused cloud identity despite individually passing proofs', async () => { const result = await runBraidSandboxSoak({ runs: 2, concurrency: 1, stressRunner: async ({ attemptIndex }) => - passedProof(attemptIndex, { - progress: { - firstControlRef: { - provider: 'tangle-sandbox', - environmentId: 'sandbox-reused', - sessionId: `session-${attemptIndex}`, - executionId: `execution-${attemptIndex}`, - runId: `provider-run-${attemptIndex}`, - requestDigest: `sha256:${'a'.repeat(64)}`, - }, - }, - }), + proofWithCloudEnvironment(attemptIndex, 'sandbox-reused'), }) assert.equal(result.status, 'failed') diff --git a/test/tangle-sandbox-braid-stress-runtime.test.mjs b/test/tangle-sandbox-braid-stress-runtime.test.mjs new file mode 100644 index 0000000..6f0bd90 --- /dev/null +++ b/test/tangle-sandbox-braid-stress-runtime.test.mjs @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + assertExactRemoteStatus, + assertRestartedCancellationRun, + assertVerifiedProcessCleanup, + closeBraidWithProof, + telemetryDisclosure, +} from '../scripts/live-required/tangle-sandbox-braid-stress.mjs' + +const controlRef = { + provider: 'tangle-sandbox', + environmentId: 'sandbox-runtime-proof', + sessionId: 'session-runtime-proof', + executionId: 'execution-runtime-proof', + runId: 'provider-run-runtime-proof', + requestDigest: `sha256:${'a'.repeat(64)}`, +} + +function environment() { + return { + id: 'environment-local-runtime-proof', + providerEnvironmentId: controlRef.environmentId, + runtimeEndpointHost: 'runtime.internal', + machineId: 'machine-runtime-proof', + placement: { region: 'us-west-2' }, + unavailableTelemetry: [], + } +} + +function terminalRun() { + return { + id: 'run-runtime-proof', + environmentId: 'environment-local-runtime-proof', + model: 'tangle-router/glm-5.2', + inputTokens: 12, + outputTokens: 3, + tokensKnown: true, + costUsd: 0.01, + usdKnown: true, + startedAt: '2026-08-12T00:00:00.000Z', + terminalAt: '2026-08-12T00:00:01.000Z', + } +} + +const workspaceVerification = { + resourceSample: { status: 'observed', value: { cpuPercent: 2 } }, +} +const account = { customerId: 'customer-proof', billingOwnerId: 'billing-proof' } + +test('remote cancellation status stays bound to one exact provider run', () => { + const status = { + status: 'cancelled', + latestExecutionId: controlRef.executionId, + runControlRef: controlRef, + } + assert.equal(assertExactRemoteStatus(status, controlRef), status) + assert.throws( + () => + assertExactRemoteStatus({ ...status, latestExecutionId: 'foreign-execution' }, controlRef), + /another execution/iu, + ) + assert.throws( + () => + assertExactRemoteStatus( + { ...status, runControlRef: { ...controlRef, runId: 'foreign-provider-run' } }, + controlRef, + ), + /changed provider control identity/iu, + ) +}) + +test('cancellation retry requires a terminal run with the saved exact control reference', () => { + const run = { status: 'cancelled', controlRef } + assert.equal(assertRestartedCancellationRun(run, controlRef), run) + assert.throws( + () => + assertRestartedCancellationRun( + { ...run, controlRef: { ...controlRef, executionId: 'foreign-execution' } }, + controlRef, + ), + /changed provider control identity/iu, + ) + assert.throws( + () => assertRestartedCancellationRun({ ...run, status: 'running' }, controlRef), + /restored running/iu, + ) +}) + +test('process proof rejects surviving descendants and an unbounded exit', () => { + const complete = { + termination: { + exited: true, + descendantsVerified: true, + cleanupStatus: 'kill', + }, + exit: { signal: 'SIGKILL', code: null }, + } + assert.deepEqual(assertVerifiedProcessCleanup(complete, 'test process'), { + cleanupStatus: 'kill', + exited: true, + descendantsVerified: true, + exit: complete.exit, + }) + assert.throws( + () => + assertVerifiedProcessCleanup( + { + ...complete, + termination: { ...complete.termination, descendantsVerified: false }, + }, + 'test process', + ), + /process-tree cleanup/iu, + ) + assert.throws( + () => assertVerifiedProcessCleanup({ ...complete, exit: { timeout: true } }, 'test process'), + /process-tree cleanup/iu, + ) +}) + +test('shutdown failures retain verified process-tree cleanup evidence', async () => { + const shutdownError = new Error('shutdown acknowledgement was lost') + const complete = { + termination: { + cleanupStatus: 'term', + exited: true, + descendantsVerified: true, + }, + exit: { code: 0, signal: null }, + } + const session = { + closed: false, + send() {}, + async waitFor() { + throw shutdownError + }, + async close() { + return complete + }, + } + + await assert.rejects( + () => closeBraidWithProof(session, 'test process'), + (error) => { + assert.equal(error, shutdownError) + assert.deepEqual(error.processCleanup, { + cleanupStatus: 'term', + exited: true, + descendantsVerified: true, + exit: complete.exit, + }) + return true + }, + ) +}) + +test('every telemetry field is observed, unavailable, provider-default, or explicitly in flight', () => { + const state = { environments: [environment()] } + const complete = telemetryDisclosure(terminalRun(), state, workspaceVerification, account) + assert.equal(complete.completeDisclosure, true) + assert.equal(complete.fields.tokens.status, 'observed') + assert.equal(complete.fields.endToEndDuration.status, 'observed') + + const inFlight = telemetryDisclosure( + { + id: 'run-runtime-proof', + environmentId: 'environment-local-runtime-proof', + model: 'tangle-router/glm-5.2', + }, + state, + workspaceVerification, + account, + { allowInFlight: true }, + ) + assert.equal(inFlight.fields.tokens.status, 'in-flight') + assert.equal(inFlight.fields.cost.status, 'in-flight') + assert.equal(inFlight.fields.endToEndDuration.status, 'in-flight') + + assert.throws( + () => + telemetryDisclosure(terminalRun(), state, { resourceSample: { status: 'missing' } }, account), + /silently missing/iu, + ) +}) diff --git a/test/tangle-sandbox-braid-stress-script.test.mjs b/test/tangle-sandbox-braid-stress-script.test.mjs index 05565e9..002aaf3 100644 --- a/test/tangle-sandbox-braid-stress-script.test.mjs +++ b/test/tangle-sandbox-braid-stress-script.test.mjs @@ -3,6 +3,7 @@ import test from 'node:test' import { cloudFailureEventTimeline, + runIdForOperation, spendDisclosure, } from '../scripts/live-required/tangle-sandbox-braid-stress.mjs' @@ -21,6 +22,7 @@ import { observationFromResponses, resourceDelta, runObservations, + stateRoundTrip, visibleEventKeys, waitForControlIdentity, waitForVisibleEvents, @@ -39,6 +41,48 @@ function event(kind, payload) { return { type: 'event', event: { kind, payload } } } +test('get_state waits for the state response instead of an acknowledgement', async () => { + let sent + const session = { + responses: [], + send(request) { + sent = request + }, + async waitFor(_label, predicate) { + const response = { type: 'state', requestId: sent.requestId, state: { runs: [] } } + assert.equal(predicate(response), true) + return response + }, + } + + const result = await stateRoundTrip(session) + assert.deepEqual(result.state, { runs: [] }) +}) + +test('cleanup recovers exactly one durable run after a lost send acknowledgement', () => { + const state = { + runs: [ + { id: 'run-other', operationId: 'operation-other' }, + { id: 'run-proof', operationId: 'operation-proof' }, + ], + } + assert.equal(runIdForOperation(state, 'operation-proof'), 'run-proof') + assert.equal(runIdForOperation(state, 'operation-missing'), undefined) + assert.throws( + () => + runIdForOperation( + { + runs: [ + { id: 'run-1', operationId: 'operation-duplicate' }, + { id: 'run-2', operationId: 'operation-duplicate' }, + ], + }, + 'operation-duplicate', + ), + /more than one Braid run/u, + ) +}) + test('extracts exact control identity and an explicit provider cursor', () => { const responses = [ event('run.environment.observed', { @@ -83,6 +127,50 @@ test('rejects visible events without stable provider identity and catches duplic assert.throws(() => assertUniqueVisibleEvents(responses, 'run-1', 'test'), /duplicate visible/iu) }) +test('rejects foreign provider run and execution identities in visible events', () => { + const observed = event('run.environment.observed', { + runId: 'run-1', + controlRef, + provider: { eventId: 'environment-event-1', providerSequence: 1 }, + }) + const visible = (identity) => + event('run.text.delta', { + runId: 'run-1', + provider: { + eventId: 'event-2', + providerSequence: 2, + cursor: 'cursor-2', + ...identity, + }, + }) + + assert.doesNotThrow(() => + assertUniqueVisibleEvents( + [observed, visible({ runId: controlRef.runId, executionId: controlRef.executionId })], + 'run-1', + 'identity', + ), + ) + assert.throws( + () => + assertUniqueVisibleEvents( + [observed, visible({ runId: 'foreign-provider-run' })], + 'run-1', + 'identity', + ), + /foreign provider runId/iu, + ) + assert.throws( + () => + assertUniqueVisibleEvents( + [observed, visible({ executionId: 'foreign-provider-execution' })], + 'run-1', + 'identity', + ), + /foreign provider executionId/iu, + ) +}) + test('never attributes an unscoped provider event to a run', () => { const responses = [ event('run.environment.observed', { value: { runId: 'run-1' } }), @@ -236,6 +324,7 @@ test('failure diagnostics retain the run boundary without exposing credentials', kind: 'run.unknown', runId: 'run-1', detail: 'HTTP 404 authorization: Bearer should-not-leak', + error: 'Invalid API key: sk-live-sentinel-123', }, }, event('run.unknown', { runId: 'run-2', detail: 'unrelated' }), @@ -249,9 +338,11 @@ test('failure diagnostics retain the run boundary without exposing credentials', kind: 'run.unknown', runId: 'run-1', detail: 'HTTP 404 authorization=[redacted]', + error: 'Invalid API key=[redacted]', }, ]) assert.equal(JSON.stringify(timeline).includes('should-not-leak'), false) + assert.equal(JSON.stringify(timeline).includes('sk-live-sentinel-123'), false) assert.equal(JSON.stringify(timeline).includes('unrelated'), false) }) @@ -305,6 +396,60 @@ test('requires fresh replay to advance beyond the persisted provider cursor', () ) }) +test('rejects a 4-to-6 replay gap and any reported missing history', () => { + const acknowledged = [ + event('run.text.delta', { + runId: 'run-1', + provider: { eventId: 'event-4', providerSequence: 4, cursor: 'cursor-4' }, + }), + ] + const skipped = [ + event('run.text.delta', { + runId: 'run-1', + provider: { eventId: 'event-6', providerSequence: 6, cursor: 'cursor-6' }, + }), + ] + + assert.throws( + () => assertProviderResumeProgress(acknowledged, skipped, 'run-1', 'cursor-4'), + /not contiguous/iu, + ) + assert.throws( + () => + assertProviderResumeProgress( + [ + ...acknowledged, + { + type: 'state', + state: { runs: [{ id: 'run-1', missingSequence: { from: 5, to: 5 } }] }, + }, + ], + [...skipped, { type: 'state', state: { missingHistory: [] } }], + 'run-1', + 'cursor-4', + ), + /missing provider history/iu, + ) + assert.throws( + () => + assertProviderResumeProgress( + acknowledged, + [ + ...skipped, + { + type: 'state', + state: { + missingHistory: [{ runId: 'run-1', fromSequence: 5, toSequence: 5 }], + }, + }, + ], + 'run-1', + 'cursor-4', + ), + /missing provider history/iu, + ) +}) + test('waits for the first stable visible event before allowing a restart snapshot', async () => { const session = { responses: [] } setTimeout(() => { diff --git a/test/w8-runs.test.ts b/test/w8-runs.test.ts index 291687e..1bb595d 100644 --- a/test/w8-runs.test.ts +++ b/test/w8-runs.test.ts @@ -33,6 +33,26 @@ function finalEvent(text: string): RuntimeStreamEvent { } } +function failedAsyncIterable(error: unknown): AsyncIterable { + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + next: async () => Promise.reject(error), + } + }, + } +} + +function emptyAsyncIterable(): AsyncIterable { + return { + [Symbol.asyncIterator](): AsyncIterator { + return { + next: async () => ({ done: true, value: undefined }), + } + }, + } +} + function appFor(execution: ExecutionPort, journal?: MemoryJournal): BraidApplication { const app = new BraidApplication({ profile: DETERMINISTIC_PROFILE, @@ -202,6 +222,54 @@ test('a disconnected live iterator reconnects and replays before declaring an un assert.equal(state.messages[1]?.text, 'after replay') }) +test('failed dispatch preserves its diagnostic when recovery finds no provider run', async () => { + const execution: ExecutionPort = { + capabilities: () => REPLAY_CAPABILITIES, + streamTurn: () => + failedAsyncIterable( + Object.assign(new Error('[{"unrecognized":"runControlRef"}]'), { + code: 'SIDECAR_SCHEMA_REJECTED', + }), + ), + reconnect: () => emptyAsyncIterable(), + status: async () => null, + } + const app = appFor(execution) + + const state = await app.send({ operationId: 'op-failed-dispatch', text: 'start once' }).completion + + assert.equal(state.runs[0]?.status, 'unknown') + assert.equal(state.lastError, 'SIDECAR_SCHEMA_REJECTED') + const unknown = app + .events() + .filter((entry) => entry.event.kind === 'run.unknown') + .at(-1)?.event + if (unknown?.kind !== 'run.unknown') throw new Error('Missing run.unknown event') + assert.equal(unknown.detail, 'SIDECAR_SCHEMA_REJECTED') +}) + +test('failed dispatch preserves its typed diagnostic when exact status is unavailable', async () => { + const execution: ExecutionPort = { + capabilities: () => ({ + ...REPLAY_CAPABILITIES, + controls: { ...REPLAY_CAPABILITIES.controls, status: false }, + }), + streamTurn: () => + failedAsyncIterable( + Object.assign(new Error('permanent cloud provisioning rejection'), { + code: 'CLOUD_PROVISION_REJECTED', + }), + ), + reconnect: () => emptyAsyncIterable(), + } + const app = appFor(execution) + + const state = await app.send({ operationId: 'op-no-status', text: 'start once' }).completion + + assert.equal(state.runs[0]?.status, 'unknown') + assert.equal(state.lastError, 'CLOUD_PROVISION_REJECTED') +}) + test('explicit cancellation is acknowledged and reaches cancelled, while legacy abort remains distinct', async () => { let release: (() => void) | undefined const execution: ExecutionPort = {