diff --git a/docs/COHORT_PROTOCOL.md b/docs/COHORT_PROTOCOL.md new file mode 100644 index 0000000..20a5a5b --- /dev/null +++ b/docs/COHORT_PROTOCOL.md @@ -0,0 +1,149 @@ +# UpdAPI Cohort Protocol + +This document defines the pre-result contract for repeated benchmark trials. The machine-readable plan authority is `schemas/cohort-plan.schema.json`; standardized isolation qualification is recorded with `schemas/isolation-canary-evidence.schema.json`. + +## Why this exists + +Repeated trials are only credible if the trial count, exact execution condition, budgets, replacement policy, exclusions, ordering and aggregation are fixed before seeing model outcomes. A cohort plan therefore acts as a preregistration record for one bounded evaluation campaign. + +## Freeze point + +The cohort plan must be schema-valid, semantically cross-validated, byte-preserved and hash-addressed before the first real agent attempt in that cohort. After that point: + +- scored failures are never replaced; +- apparatus-invalid attempts remain visible and may be replaced only within the predeclared cap; +- material product/model/tool/network/context/isolation changes abort the cohort or create a new cohort; +- post-hoc exclusions, schedule edits or aggregation changes are forbidden. + +The orchestrator must copy the exact frozen plan bytes and SHA-256 into cohort evidence. A later file with the same logical fields but different bytes is not the frozen plan. + +## Standardized agent cells + +Every real-agent cell must bind to: + +- the exact benchmark commit and clean-tree requirement; +- adapter source hash; +- exact expected product/CLI version and requested model configuration; +- case ID/version and declared condition; +- sterile agent context mode (`bare` or `sterile_config`); +- normalized invocation-template hash; +- normalized context/settings-profile hash; +- explicit web, repository, execution, package-install, retrieval-MCP and external-network capabilities; +- normalized configured-MCP hash (or `none`); +- OS-enforced isolation profile hash; +- retained isolation-canary evidence hash and pass timestamp; +- explicit timeout, turn, token and cost budgets (`null` means deliberately unsupported/unbounded; omission is forbidden); +- target scored-attempt count and invalid-replacement cap. + +A host-context development run cannot satisfy the standardized cell contract. + +## Isolation qualification + +The machine-readable canary record must contain exactly two fresh-nonce passes. Each pass must prove: + +1. built-in file tools can read an allowed workspace nonce; +2. shell/subprocess tools can read an allowed workspace nonce; +3. built-in file tools cannot reveal a forbidden answer-bearing nonce; +4. shell/subprocess tools cannot reveal that forbidden nonce; +5. a context nonce placed in throwaway host configuration is absent from the standardized launch; +6. deliberate isolation-unavailable startup fails closed rather than silently running unsandboxed. + +The plaintext nonces need not be retained after qualification; their SHA-256 values and raw tool-evidence artifact hashes are retained. + +### Required cross-file equality + +JSON Schema validates document shape, but the orchestrator must additionally reject a cohort plan unless its referenced canary evidence matches the cell exactly on: + +- adapter name and source SHA-256; +- expected product/CLI version; +- model configuration; +- `agent_context_mode`; +- `execution_policy.context_profile_sha256`; +- the SHA-256 of the normalized execution policy; +- isolation mode and isolation profile SHA-256; +- qualification status `passed` with two valid fresh-nonce passes. + +A canary produced under a different adapter, CLI version, model configuration, context profile, tool/network policy or sandbox profile cannot qualify the cohort merely because its evidence file has a valid hash. + +## Execution-policy normalization + +Secrets must never enter the plan or hashes. The implementation should create a deterministic non-secret execution-policy record and hash canonical bytes. It must cover at least: + +- exact CLI/invocation template, replacing per-run workspace paths/IDs with documented placeholders; +- requested model/configuration flags; +- permission/tool flags; +- web/network policy; +- MCP server identities/configuration with secret values removed; +- normalized context/settings profile. + +The run manifest remains observed provenance. The orchestrator must compare each observed run against the frozen cell and classify a material mismatch as apparatus-invalid / cohort-aborting according to the committed policy rather than silently mixing configurations. + +## Attempt ordering + +A seed alone is insufficient preregistration because scheduler implementations can change. The cohort plan therefore stores: + +- a versioned schedule algorithm; +- its seed; +- the exact generated ordered candidate schedule. + +The generated schedule is the execution authority. Primary slots are always eligible. Invalid-replacement slots execute only when a prior apparatus-invalid attempt in that cell requires replacement; otherwise they are retained as unused/skipped slots. The orchestrator must validate that primary/replacement counts agree with each cell's target and replacement cap. + +For multi-cell cohorts, prefer randomized/interleaved primary slots to reduce temporal product-update confounding. For the initial one-cell MCP development cohort, explicit sequential order is acceptable because interleaving has no effect. + +## Round-4 development cohort + +The first real repeated cohort remains intentionally small: + +- one cell; +- `case-mcp-modern-era-negotiation-v2`; +- Claude Code adapter only; +- `agent_default` standardized condition; +- exactly 5 scored attempts targeted; +- at most 2 apparatus-invalid replacements; +- every invalid attempt retained; +- no scored failure replacement; +- pre-release/development label only. + +Before attempt 1, the exact timeout/turn/token/cost budget vector, execution policy, model configuration, isolation qualification and all 5 primary + 2 conditional replacement schedule slots must already exist in the frozen plan. + +## Aggregation contract + +The long-run reporting hierarchy is: + +`attempt -> case -> API change event -> provider/ecosystem family -> overall` + +This prevents a provider or release with many authored cases from dominating the headline score. The planned headline aggregation is a macro-average over provider/ecosystem families, with raw success rates and category slices also shown. + +Apparatus-invalid attempts are excluded from capability scores but reported. A benchmark integrity violation by the evaluated agent, such as retrieving UpdAPI's answer/reference material, is not apparatus-invalid: it is a scored failure and is reported separately. + +## No hidden retry semantics + +Product-native retries inside one agent invocation are part of that attempt. Harness-level retries are separate attempts and must follow the frozen candidate schedule and replacement policy. No benchmark runner, future scheduler or XUXI supervisor may silently retry a scored failure under the same or a new attempt identity. + +## Required semantic validation before launch + +Schema validity alone does not authorize a cohort. The orchestrator must refuse real-agent launch unless all of the following hold: + +- benchmark worktree is clean and at the exact frozen commit; +- case/version exists at that commit; +- adapter source hash matches; +- product version/model preflight matches the frozen cell where observable before invocation; +- execution-policy hashes/configuration match; +- canary evidence is schema-valid and cross-file-compatible as defined above; +- generated schedule counts match target attempts and replacement caps; +- the exact frozen plan bytes/hash are stored before attempt 1. + +Failure of any preflight condition is apparatus failure, not a model score. + +The repository exposes the host-independent portion of this gate as: + +```sh +npm run bench:cohort:validate -- path/to/plan.json \ + --canary path/to/isolation-canary.json +``` + +The command schema-validates the plan/evidence, checks schedule counts, binds each agent cell to canary evidence by exact SHA-256, verifies adapter/product/model/context/execution/isolation equality, and prints the SHA-256 of the exact plan bytes. The repeated-trial orchestrator should call the same library/gate rather than reimplement weaker checks. + +## Portability + +The cohort plan is independent of GitHub Actions and must be locally executable. Future XUXI supervision may schedule or launch the same cohort semantics, but may not alter the frozen plan after results begin. diff --git a/package.json b/package.json index 146a475..38f50c9 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "bench:validate": "node tools/bench/validate-data.mjs", "bench:controls": "node tools/bench/run-controls.mjs", "bench:run": "node tools/bench/run-benchmark.mjs", + "bench:cohort:validate": "node tools/bench/validate-cohort.mjs", "bench:evidence": "node tools/bench/capture-registry-evidence.mjs", "check-links": "node tools/check-links.mjs", "check-links:fix": "node tools/check-links.mjs --fix", diff --git a/schemas/cohort-plan.schema.json b/schemas/cohort-plan.schema.json new file mode 100644 index 0000000..c0c427f --- /dev/null +++ b/schemas/cohort-plan.schema.json @@ -0,0 +1,304 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/cohort-plan.schema.json", + "title": "UpdAPI preregistered cohort plan", + "description": "Immutable-before-results contract for a repeated-trial benchmark cohort. The plan fixes membership, execution policy, budgets, exact attempt schedule, replacement policy, isolation qualification and aggregation before the first scored agent attempt. schema_version 0.2.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "cohort_id", + "plan_version", + "created_at", + "benchmark", + "cells", + "budgets", + "ordering", + "replacement_policy", + "aggregation", + "planned_exclusions", + "freeze_policy" + ], + "properties": { + "schema_version": { "const": "0.2.0" }, + "cohort_id": { + "type": "string", + "minLength": 8, + "pattern": "^[a-z0-9][a-z0-9._-]+$" + }, + "plan_version": { "type": "integer", "minimum": 1 }, + "created_at": { "type": "string", "format": "date-time" }, + "description": { "type": "string" }, + "benchmark": { + "type": "object", + "additionalProperties": false, + "required": ["commit", "clean_tree_required"], + "properties": { + "commit": { + "type": "string", + "pattern": "^[0-9a-f]{40}$", + "description": "Exact UpdAPI commit that defines the cases, runner, validators, schemas and adapter used by this cohort." + }, + "clean_tree_required": { "const": true }, + "benchmark_version": { "type": "string" } + } + }, + "cells": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "cell_id", + "case_id", + "case_version", + "condition", + "adapter", + "agent_context_mode", + "execution_policy", + "isolation", + "target_scored_attempts", + "max_invalid_replacements" + ], + "properties": { + "cell_id": { "type": "string", "minLength": 4 }, + "case_id": { "type": "string", "minLength": 4 }, + "case_version": { "type": "integer", "minimum": 1 }, + "condition": { + "enum": [ + "agent_default", + "no_web", + "no_external_retrieval", + "authoritative_docs_supplied", + "updapi_mcp", + "no_execution", + "control_stale", + "control_current", + "control_human_fixed" + ] + }, + "adapter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "source_sha256", + "expected_product", + "expected_product_version", + "model_configuration" + ], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "source_sha256": { "$ref": "#/$defs/sha256" }, + "expected_product": { "type": "string", "minLength": 1 }, + "expected_product_version": { "type": "string", "minLength": 1 }, + "model_configuration": { + "type": "string", + "minLength": 1, + "description": "Exact requested model/configuration identifier, or an explicit product-default token if that is the preregistered condition. A material observed change aborts/splits the cohort." + } + } + }, + "agent_context_mode": { + "enum": ["bare", "sterile_config", "not_applicable"], + "description": "Standardized benchmark cells use bare or sterile_config. Controls use not_applicable." + }, + "execution_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "invocation_template_sha256", + "context_profile_sha256", + "web_search", + "repository_access", + "execution", + "package_install", + "retrieval_mcp", + "external_network", + "configured_mcp_sha256" + ], + "properties": { + "invocation_template_sha256": { + "$ref": "#/$defs/sha256", + "description": "Hash of the normalized exact CLI/invocation template after redacting secret values and replacing per-run paths/IDs with canonical placeholders." + }, + "context_profile_sha256": { + "$ref": "#/$defs/sha256", + "description": "Hash of the normalized non-secret agent context/settings profile." + }, + "web_search": { "$ref": "#/$defs/capability" }, + "repository_access": { "$ref": "#/$defs/capability" }, + "execution": { "$ref": "#/$defs/capability" }, + "package_install": { "$ref": "#/$defs/capability" }, + "retrieval_mcp": { "$ref": "#/$defs/capability" }, + "external_network": { "$ref": "#/$defs/capability" }, + "configured_mcp_sha256": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "const": "none" } + ], + "description": "Hash of normalized configured MCP server identities/configuration with secrets removed, or none." + }, + "notes": { "type": "string" } + } + }, + "isolation": { + "type": "object", + "additionalProperties": false, + "required": ["mode"], + "properties": { + "mode": { + "enum": ["wsl2_claude_sandbox", "container", "vm", "none"], + "description": "OS-enforced execution boundary for agent cells; none is valid only for controls." + }, + "profile_sha256": { "$ref": "#/$defs/sha256" }, + "canary_evidence_sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of a schema-valid isolation-canary evidence record qualified for the exact adapter/context/execution/isolation profile." + }, + "canary_passed_at": { "type": "string", "format": "date-time" } + } + }, + "target_scored_attempts": { "type": "integer", "minimum": 1 }, + "max_invalid_replacements": { "type": "integer", "minimum": 0 }, + "notes": { "type": "string" } + }, + "allOf": [ + { + "if": { + "properties": { + "condition": { + "enum": [ + "agent_default", + "no_web", + "no_external_retrieval", + "authoritative_docs_supplied", + "updapi_mcp", + "no_execution" + ] + } + }, + "required": ["condition"] + }, + "then": { + "properties": { + "agent_context_mode": { "enum": ["bare", "sterile_config"] }, + "isolation": { + "properties": { + "mode": { "enum": ["wsl2_claude_sandbox", "container", "vm"] }, + "profile_sha256": { "$ref": "#/$defs/sha256" }, + "canary_evidence_sha256": { "$ref": "#/$defs/sha256" }, + "canary_passed_at": { "type": "string", "format": "date-time" } + }, + "required": ["mode", "profile_sha256", "canary_evidence_sha256", "canary_passed_at"] + } + } + }, + "else": { + "properties": { + "agent_context_mode": { "const": "not_applicable" }, + "isolation": { + "properties": { "mode": { "const": "none" } }, + "required": ["mode"] + } + } + } + } + ] + } + }, + "budgets": { + "type": "object", + "additionalProperties": false, + "required": ["timeout_ms", "max_turns", "max_tokens", "max_cost_usd"], + "properties": { + "timeout_ms": { "type": "integer", "minimum": 1000 }, + "max_turns": { "type": ["integer", "null"], "minimum": 1 }, + "max_tokens": { "type": ["integer", "null"], "minimum": 1 }, + "max_cost_usd": { "type": ["number", "null"], "minimum": 0 } + }, + "description": "Every budget dimension is explicit. null means deliberately unsupported/unbounded for this cohort; omission is not permitted." + }, + "ordering": { + "type": "object", + "additionalProperties": false, + "required": ["strategy", "algorithm", "seed", "generated_schedule"], + "properties": { + "strategy": { "enum": ["randomized_interleaved", "randomized", "sequential"] }, + "algorithm": { + "enum": ["sha256_fisher_yates_v1", "explicit_v1"], + "description": "Versioned deterministic schedule-generation algorithm. The generated schedule remains the execution authority." + }, + "seed": { "type": "string", "minLength": 1 }, + "generated_schedule": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["ordinal", "cell_id", "slot_kind", "slot_index"], + "properties": { + "ordinal": { "type": "integer", "minimum": 1 }, + "cell_id": { "type": "string", "minLength": 4 }, + "slot_kind": { "enum": ["primary", "invalid_replacement"] }, + "slot_index": { "type": "integer", "minimum": 1 } + } + }, + "description": "Exact ordered candidate attempt slots. Replacement slots execute only when a prior apparatus-invalid attempt in that cell requires replacement; otherwise they are skipped and recorded as unused." + }, + "description": { "type": "string" } + } + }, + "replacement_policy": { + "type": "object", + "additionalProperties": false, + "required": ["replace_apparatus_invalid", "replace_scored_failure", "retain_invalid_attempts", "on_replacement_cap_exceeded"], + "properties": { + "replace_apparatus_invalid": { "const": true }, + "replace_scored_failure": { "const": false }, + "retain_invalid_attempts": { "const": true }, + "on_replacement_cap_exceeded": { "const": "mark_cell_incomplete" } + } + }, + "aggregation": { + "type": "object", + "additionalProperties": false, + "required": ["attempt_to_case", "case_to_event", "event_to_family", "family_to_overall", "invalid_handling", "integrity_violation_handling"], + "properties": { + "attempt_to_case": { "const": "mean_verified_success_over_scored_attempts" }, + "case_to_event": { "const": "mean_case_score" }, + "event_to_family": { "const": "mean_event_score" }, + "family_to_overall": { "const": "macro_mean_family_score" }, + "invalid_handling": { "const": "exclude_from_capability_score_but_report" }, + "integrity_violation_handling": { "const": "scored_failure_and_report" } + } + }, + "planned_exclusions": { + "type": "array", + "items": { "enum": ["apparatus_invalid"] }, + "uniqueItems": true, + "maxItems": 1, + "description": "No post-hoc exclusions are permitted. For v0, only attempts classified apparatus-invalid by the committed result contract may be excluded from capability scoring." + }, + "freeze_policy": { + "type": "object", + "additionalProperties": false, + "required": ["immutable_before_first_agent_attempt", "material_config_change_action", "posthoc_plan_edits"], + "properties": { + "immutable_before_first_agent_attempt": { "const": true }, + "material_config_change_action": { "const": "abort_or_new_cohort" }, + "posthoc_plan_edits": { "const": "forbidden" } + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "capability": { + "enum": ["allowed", "denied", "unavailable"] + } + } +} diff --git a/schemas/isolation-canary-evidence.schema.json b/schemas/isolation-canary-evidence.schema.json new file mode 100644 index 0000000..c56f20e --- /dev/null +++ b/schemas/isolation-canary-evidence.schema.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/isolation-canary-evidence.schema.json", + "title": "UpdAPI isolation canary evidence", + "description": "Machine-readable qualification evidence for one standardized agent execution profile. Two fresh-nonce passes must prove allowed workspace reads work, answer-bearing paths are denied through built-in and shell channels, inherited host context is absent, and isolation fails closed when unavailable. schema_version 0.1.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "canary_id", + "created_at", + "adapter", + "agent_context_mode", + "context_profile_sha256", + "execution_policy_sha256", + "isolation", + "passes", + "qualification" + ], + "properties": { + "schema_version": { "const": "0.1.0" }, + "canary_id": { + "type": "string", + "minLength": 8, + "pattern": "^[a-z0-9][a-z0-9._-]+$" + }, + "created_at": { "type": "string", "format": "date-time" }, + "adapter": { + "type": "object", + "additionalProperties": false, + "required": ["name", "source_sha256", "product", "product_version", "model_configuration"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "source_sha256": { "$ref": "#/$defs/sha256" }, + "product": { "type": "string", "minLength": 1 }, + "product_version": { "type": "string", "minLength": 1 }, + "model_configuration": { "type": "string", "minLength": 1 } + } + }, + "agent_context_mode": { + "enum": ["bare", "sterile_config"] + }, + "context_profile_sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the normalized non-secret agent context/configuration profile used for the canary." + }, + "execution_policy_sha256": { + "$ref": "#/$defs/sha256", + "description": "SHA-256 of the normalized invocation/tool/network policy used for both the canary and the benchmark cohort." + }, + "isolation": { + "type": "object", + "additionalProperties": false, + "required": ["mode", "profile_sha256", "fail_closed"], + "properties": { + "mode": { "enum": ["wsl2_claude_sandbox", "container", "vm"] }, + "profile_sha256": { "$ref": "#/$defs/sha256" }, + "fail_closed": { "const": true } + } + }, + "passes": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "pass_id", + "executed_at", + "allowed_nonce_sha256", + "forbidden_nonce_sha256", + "context_nonce_sha256", + "checks", + "artifact_index_sha256" + ], + "properties": { + "pass_id": { "type": "string", "minLength": 1 }, + "executed_at": { "type": "string", "format": "date-time" }, + "allowed_nonce_sha256": { "$ref": "#/$defs/sha256" }, + "forbidden_nonce_sha256": { "$ref": "#/$defs/sha256" }, + "context_nonce_sha256": { "$ref": "#/$defs/sha256" }, + "checks": { + "type": "object", + "additionalProperties": false, + "required": [ + "allowed_builtin_read", + "allowed_shell_read", + "forbidden_builtin_read_denied", + "forbidden_shell_read_denied", + "context_nonce_absent", + "isolation_unavailable_fails_closed" + ], + "properties": { + "allowed_builtin_read": { "const": true }, + "allowed_shell_read": { "const": true }, + "forbidden_builtin_read_denied": { "const": true }, + "forbidden_shell_read_denied": { "const": true }, + "context_nonce_absent": { "const": true }, + "isolation_unavailable_fails_closed": { "const": true } + } + }, + "artifact_index_sha256": { + "$ref": "#/$defs/sha256", + "description": "Hash of the retained artifact index containing raw tool attempts/outputs and their per-file hashes." + }, + "notes": { "type": "string" } + } + } + }, + "qualification": { + "type": "object", + "additionalProperties": false, + "required": ["status", "qualified_at"], + "properties": { + "status": { "const": "passed" }, + "qualified_at": { "type": "string", "format": "date-time" } + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/test/cohort-contract.test.cjs b/test/cohort-contract.test.cjs new file mode 100644 index 0000000..7971ce7 --- /dev/null +++ b/test/cohort-contract.test.cjs @@ -0,0 +1,223 @@ +const assert = require('node:assert'); +const { createHash } = require('node:crypto'); + +describe('bench cohort preregistration contract', function () { + let validators; + let assertValid; + let assertCohortSemantics; + let assertCanaryQualifiesCell; + let sha256Canonical; + + before(async function () { + ({ validators, assertValid } = await import('../tools/bench/lib/schemas.mjs')); + ({ assertCohortSemantics, assertCanaryQualifiesCell, sha256Canonical } = await import('../tools/bench/lib/cohort.mjs')); + }); + + const h64 = (char) => char.repeat(64); + const h40 = (char) => char.repeat(40); + + function makeCell() { + return { + cell_id: 'mcp-claude-default', + case_id: 'case-mcp-modern-era-negotiation-v2', + case_version: 1, + condition: 'agent_default', + adapter: { + name: 'claude-code', + source_sha256: h64('a'), + expected_product: 'claude-code', + expected_product_version: '1.2.3', + model_configuration: 'claude-opus-5' + }, + agent_context_mode: 'sterile_config', + execution_policy: { + invocation_template_sha256: h64('b'), + context_profile_sha256: h64('c'), + web_search: 'allowed', + repository_access: 'allowed', + execution: 'allowed', + package_install: 'allowed', + retrieval_mcp: 'denied', + external_network: 'allowed', + configured_mcp_sha256: 'none' + }, + isolation: { + mode: 'wsl2_claude_sandbox', + profile_sha256: h64('d'), + canary_evidence_sha256: h64('e'), + canary_passed_at: '2026-08-29T08:00:00Z' + }, + target_scored_attempts: 5, + max_invalid_replacements: 2 + }; + } + + function makePlan(cell = makeCell()) { + return { + schema_version: '0.2.0', + cohort_id: 'round4-mcp-claude', + plan_version: 1, + created_at: '2026-08-29T08:05:00Z', + benchmark: { + commit: h40('f'), + clean_tree_required: true, + benchmark_version: 'v0-dev' + }, + cells: [cell], + budgets: { + timeout_ms: 300000, + max_turns: 30, + max_tokens: null, + max_cost_usd: 5 + }, + ordering: { + strategy: 'sequential', + algorithm: 'explicit_v1', + seed: 'round4-fixed-seed', + generated_schedule: [ + { ordinal: 1, cell_id: cell.cell_id, slot_kind: 'primary', slot_index: 1 }, + { ordinal: 2, cell_id: cell.cell_id, slot_kind: 'primary', slot_index: 2 }, + { ordinal: 3, cell_id: cell.cell_id, slot_kind: 'primary', slot_index: 3 }, + { ordinal: 4, cell_id: cell.cell_id, slot_kind: 'primary', slot_index: 4 }, + { ordinal: 5, cell_id: cell.cell_id, slot_kind: 'primary', slot_index: 5 }, + { ordinal: 6, cell_id: cell.cell_id, slot_kind: 'invalid_replacement', slot_index: 1 }, + { ordinal: 7, cell_id: cell.cell_id, slot_kind: 'invalid_replacement', slot_index: 2 } + ] + }, + replacement_policy: { + replace_apparatus_invalid: true, + replace_scored_failure: false, + retain_invalid_attempts: true, + on_replacement_cap_exceeded: 'mark_cell_incomplete' + }, + aggregation: { + attempt_to_case: 'mean_verified_success_over_scored_attempts', + case_to_event: 'mean_case_score', + event_to_family: 'mean_event_score', + family_to_overall: 'macro_mean_family_score', + invalid_handling: 'exclude_from_capability_score_but_report', + integrity_violation_handling: 'scored_failure_and_report' + }, + planned_exclusions: ['apparatus_invalid'], + freeze_policy: { + immutable_before_first_agent_attempt: true, + material_config_change_action: 'abort_or_new_cohort', + posthoc_plan_edits: 'forbidden' + } + }; + } + + function makeCanary(cell) { + return { + schema_version: '0.1.0', + canary_id: 'round4-isolation-canary', + created_at: '2026-08-29T07:55:00Z', + adapter: { + name: cell.adapter.name, + source_sha256: cell.adapter.source_sha256, + product: cell.adapter.expected_product, + product_version: cell.adapter.expected_product_version, + model_configuration: cell.adapter.model_configuration + }, + agent_context_mode: cell.agent_context_mode, + context_profile_sha256: cell.execution_policy.context_profile_sha256, + execution_policy_sha256: sha256Canonical(cell.execution_policy), + isolation: { + mode: cell.isolation.mode, + profile_sha256: cell.isolation.profile_sha256, + fail_closed: true + }, + passes: [1, 2].map((n) => ({ + pass_id: `nonce-pass-${n}`, + executed_at: `2026-08-29T07:5${n}:00Z`, + allowed_nonce_sha256: n === 1 ? h64('1') : h64('4'), + forbidden_nonce_sha256: n === 1 ? h64('2') : h64('5'), + context_nonce_sha256: n === 1 ? h64('3') : h64('6'), + checks: { + allowed_builtin_read: true, + allowed_shell_read: true, + forbidden_builtin_read_denied: true, + forbidden_shell_read_denied: true, + context_nonce_absent: true, + isolation_unavailable_fails_closed: true + }, + artifact_index_sha256: n === 1 ? h64('7') : h64('8') + })), + qualification: { + status: 'passed', + qualified_at: cell.isolation.canary_passed_at + } + }; + } + + it('accepts a fully frozen N=5 + 2 replacement development plan', function () { + const plan = makePlan(); + assertValid(validators.cohortPlan, plan, 'cohort plan'); + assert.strictEqual(assertCohortSemantics(plan), true); + }); + + it('rejects an omitted budget dimension instead of treating it as implicit', function () { + const plan = makePlan(); + delete plan.budgets.max_cost_usd; + assert.throws(() => assertValid(validators.cohortPlan, plan, 'cohort plan'), /max_cost_usd|required property/); + }); + + it('rejects a schedule whose primary count does not match the preregistered target', function () { + const plan = makePlan(); + plan.ordering.generated_schedule = plan.ordering.generated_schedule.filter((slot) => !(slot.slot_kind === 'primary' && slot.slot_index === 5)); + plan.ordering.generated_schedule.forEach((slot, index) => { slot.ordinal = index + 1; }); + assertValid(validators.cohortPlan, plan, 'cohort plan'); + assert.throws(() => assertCohortSemantics(plan), /primary schedule slots 4 != target_scored_attempts 5/); + }); + + it('rejects duplicate or missing logical slot indices even when schedule counts still match', function () { + const plan = makePlan(); + plan.ordering.generated_schedule[1].slot_index = 1; + assertValid(validators.cohortPlan, plan, 'cohort plan'); + assert.throws( + () => assertCohortSemantics(plan), + /primary slot_index set \[1,3,4,5\] != expected \[1,2,3,4,5\]/ + ); + }); + + it('requires a standardized agent cell rather than host-context execution', function () { + const plan = makePlan(); + plan.cells[0].agent_context_mode = 'host_context'; + assert.throws(() => assertValid(validators.cohortPlan, plan, 'cohort plan'), /agent_context_mode|allowed values/); + }); + + it('accepts two-pass machine-readable isolation evidence and binds it to the exact cell', function () { + const cell = makeCell(); + const evidence = makeCanary(cell); + const evidenceBytes = Buffer.from(JSON.stringify(evidence)); + cell.isolation.canary_evidence_sha256 = createHash('sha256').update(evidenceBytes).digest('hex'); + + assertValid(validators.isolationCanaryEvidence, evidence, 'isolation canary evidence'); + assert.strictEqual(assertCanaryQualifiesCell(cell, evidence, evidenceBytes), true); + }); + + it('rejects a duplicated first canary pass instead of accepting fake two-pass freshness', function () { + const cell = makeCell(); + const evidence = makeCanary(cell); + evidence.passes[1] = structuredClone(evidence.passes[0]); + + assertValid(validators.isolationCanaryEvidence, evidence, 'isolation canary evidence'); + assert.throws(() => assertCanaryQualifiesCell(cell, evidence), /canary passes must be independent/); + }); + + it('rejects canary evidence from a different execution policy even when both records are schema-valid', function () { + const cell = makeCell(); + const evidence = makeCanary(cell); + evidence.execution_policy_sha256 = h64('9'); + assertValid(validators.isolationCanaryEvidence, evidence, 'isolation canary evidence'); + assert.throws(() => assertCanaryQualifiesCell(cell, evidence), /execution_policy_sha256 mismatch/); + }); + + it('rejects canary evidence from a different CLI version', function () { + const cell = makeCell(); + const evidence = makeCanary(cell); + evidence.adapter.product_version = '1.2.4'; + assertValid(validators.isolationCanaryEvidence, evidence, 'isolation canary evidence'); + assert.throws(() => assertCanaryQualifiesCell(cell, evidence), /adapter.product_version mismatch/); + }); +}); diff --git a/tools/bench/lib/cohort.mjs b/tools/bench/lib/cohort.mjs new file mode 100644 index 0000000..fc6a737 --- /dev/null +++ b/tools/bench/lib/cohort.mjs @@ -0,0 +1,135 @@ +import { createHash } from 'node:crypto'; + +function canonicalize(value) { + if (Array.isArray(value)) return value.map(canonicalize); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]) + ); + } + return value; +} + +export function canonicalJson(value) { + return JSON.stringify(canonicalize(value)); +} + +export function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +export function sha256Canonical(value) { + return sha256(canonicalJson(value)); +} + +export function assertCohortSemantics(plan) { + const cells = new Map(); + for (const cell of plan.cells) { + if (cells.has(cell.cell_id)) throw new Error(`duplicate cell_id: ${cell.cell_id}`); + cells.set(cell.cell_id, cell); + } + + const seenOrdinals = new Set(); + const counts = new Map([...cells.keys()].map((id) => [id, { primary: 0, invalid_replacement: 0 }])); + const slotIndices = new Map( + [...cells.keys()].map((id) => [id, { primary: new Set(), invalid_replacement: new Set() }]) + ); + + plan.ordering.generated_schedule.forEach((slot, index) => { + const expectedOrdinal = index + 1; + if (slot.ordinal !== expectedOrdinal) { + throw new Error(`schedule ordinal ${slot.ordinal} must equal contiguous position ${expectedOrdinal}`); + } + if (seenOrdinals.has(slot.ordinal)) throw new Error(`duplicate schedule ordinal: ${slot.ordinal}`); + seenOrdinals.add(slot.ordinal); + if (!cells.has(slot.cell_id)) throw new Error(`schedule references unknown cell_id: ${slot.cell_id}`); + counts.get(slot.cell_id)[slot.slot_kind] += 1; + slotIndices.get(slot.cell_id)[slot.slot_kind].add(slot.slot_index); + }); + + for (const [id, cell] of cells) { + const count = counts.get(id); + if (count.primary !== cell.target_scored_attempts) { + throw new Error(`${id}: primary schedule slots ${count.primary} != target_scored_attempts ${cell.target_scored_attempts}`); + } + if (count.invalid_replacement !== cell.max_invalid_replacements) { + throw new Error(`${id}: replacement schedule slots ${count.invalid_replacement} != max_invalid_replacements ${cell.max_invalid_replacements}`); + } + + for (const [slotKind, expectedCount] of [ + ['primary', cell.target_scored_attempts], + ['invalid_replacement', cell.max_invalid_replacements] + ]) { + const observed = slotIndices.get(id)[slotKind]; + const expected = Array.from({ length: expectedCount }, (_, index) => index + 1); + const observedSorted = [...observed].sort((a, b) => a - b); + if ( + observedSorted.length !== expected.length + || observedSorted.some((value, index) => value !== expected[index]) + ) { + throw new Error( + `${id}: ${slotKind} slot_index set ${JSON.stringify(observedSorted)} != expected ${JSON.stringify(expected)}` + ); + } + } + } + + if (plan.ordering.strategy === 'sequential' && plan.ordering.algorithm !== 'explicit_v1') { + throw new Error('sequential ordering must use explicit_v1'); + } + if (plan.ordering.strategy !== 'sequential' && plan.ordering.algorithm !== 'sha256_fisher_yates_v1') { + throw new Error('randomized ordering must use sha256_fisher_yates_v1'); + } + + return true; +} + +export function assertCanaryQualifiesCell(cell, evidence, evidenceBytes = null) { + const mismatch = (label, expected, actual) => { + if (expected !== actual) throw new Error(`${label} mismatch: expected ${expected}, got ${actual}`); + }; + + mismatch('adapter.name', cell.adapter.name, evidence.adapter.name); + mismatch('adapter.source_sha256', cell.adapter.source_sha256, evidence.adapter.source_sha256); + mismatch('adapter.product', cell.adapter.expected_product, evidence.adapter.product); + mismatch('adapter.product_version', cell.adapter.expected_product_version, evidence.adapter.product_version); + mismatch('adapter.model_configuration', cell.adapter.model_configuration, evidence.adapter.model_configuration); + mismatch('agent_context_mode', cell.agent_context_mode, evidence.agent_context_mode); + mismatch( + 'context_profile_sha256', + cell.execution_policy.context_profile_sha256, + evidence.context_profile_sha256 + ); + mismatch( + 'execution_policy_sha256', + sha256Canonical(cell.execution_policy), + evidence.execution_policy_sha256 + ); + mismatch('isolation.mode', cell.isolation.mode, evidence.isolation.mode); + mismatch('isolation.profile_sha256', cell.isolation.profile_sha256, evidence.isolation.profile_sha256); + mismatch('canary_passed_at', cell.isolation.canary_passed_at, evidence.qualification.qualified_at); + + if (evidence.qualification.status !== 'passed') throw new Error('canary qualification is not passed'); + if (evidence.passes.length !== 2) throw new Error('canary qualification must contain exactly two passes'); + + const [firstPass, secondPass] = evidence.passes; + for (const field of [ + 'pass_id', + 'executed_at', + 'allowed_nonce_sha256', + 'forbidden_nonce_sha256', + 'context_nonce_sha256', + 'artifact_index_sha256' + ]) { + if (firstPass[field] === secondPass[field]) { + throw new Error(`canary passes must be independent: duplicate ${field}`); + } + } + + if (evidenceBytes !== null) { + const observedEvidenceSha = sha256(evidenceBytes); + mismatch('canary_evidence_sha256', cell.isolation.canary_evidence_sha256, observedEvidenceSha); + } + + return true; +} diff --git a/tools/bench/lib/schemas.mjs b/tools/bench/lib/schemas.mjs index 6ee929e..2fbd9b9 100644 --- a/tools/bench/lib/schemas.mjs +++ b/tools/bench/lib/schemas.mjs @@ -18,7 +18,9 @@ export const validators = { changeEvent: compile('change-event.schema.json'), benchmarkCase: compile('benchmark-case.schema.json'), runManifest: compile('run-manifest.schema.json'), - runResult: compile('run-result.schema.json') + runResult: compile('run-result.schema.json'), + cohortPlan: compile('cohort-plan.schema.json'), + isolationCanaryEvidence: compile('isolation-canary-evidence.schema.json') }; export function assertValid(validator, data, label) { diff --git a/tools/bench/validate-cohort.mjs b/tools/bench/validate-cohort.mjs new file mode 100644 index 0000000..fdac9c2 --- /dev/null +++ b/tools/bench/validate-cohort.mjs @@ -0,0 +1,63 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import { assertValid, validators } from './lib/schemas.mjs'; +import { assertCohortSemantics, assertCanaryQualifiesCell, sha256 } from './lib/cohort.mjs'; + +function usage() { + console.error('Usage: node tools/bench/validate-cohort.mjs [--canary ]...'); + process.exit(2); +} + +const args = process.argv.slice(2); +if (args.length < 1 || args[0].startsWith('-')) usage(); + +const planPath = path.resolve(args.shift()); +const canaryPaths = []; +while (args.length) { + const flag = args.shift(); + if (flag !== '--canary' || args.length === 0) usage(); + canaryPaths.push(path.resolve(args.shift())); +} + +try { + const planBytes = fs.readFileSync(planPath); + const plan = JSON.parse(planBytes.toString('utf8')); + assertValid(validators.cohortPlan, plan, 'cohort plan'); + assertCohortSemantics(plan); + + const canariesBySha = new Map(); + for (const canaryPath of canaryPaths) { + const bytes = fs.readFileSync(canaryPath); + const digest = sha256(bytes); + if (canariesBySha.has(digest)) throw new Error(`duplicate canary evidence hash supplied: ${digest}`); + const evidence = JSON.parse(bytes.toString('utf8')); + assertValid(validators.isolationCanaryEvidence, evidence, `isolation canary ${canaryPath}`); + canariesBySha.set(digest, { evidence, bytes, canaryPath }); + } + + const agentConditions = new Set([ + 'agent_default', + 'no_web', + 'no_external_retrieval', + 'authoritative_docs_supplied', + 'updapi_mcp', + 'no_execution' + ]); + + for (const cell of plan.cells) { + if (!agentConditions.has(cell.condition)) continue; + const digest = cell.isolation.canary_evidence_sha256; + const found = canariesBySha.get(digest); + if (!found) { + throw new Error(`${cell.cell_id}: missing --canary evidence with preregistered SHA-256 ${digest}`); + } + assertCanaryQualifiesCell(cell, found.evidence, found.bytes); + } + + const planSha = sha256(planBytes); + console.log(`OK cohort ${plan.cohort_id} plan_sha256=${planSha} cells=${plan.cells.length} schedule_slots=${plan.ordering.generated_schedule.length} canaries=${canariesBySha.size}`); +} catch (error) { + console.error(`INVALID cohort: ${error.message}`); + process.exit(1); +}