From b0e20925f170e8137ab040438262e856675c8550 Mon Sep 17 00:00:00 2001 From: Sean Phan Date: Sun, 19 Jul 2026 03:17:00 -0700 Subject: [PATCH 1/4] feat: add Composer Arena public preview --- package.json | 3 +- .../composer-arena/v0/battle.schema.json | 63 +++ .../composer-arena/v0/battles.preview.jsonl | 2 + public/bench/composer-arena/v0/checksums.json | 11 + public/bench/composer-arena/v0/handoff.md | 50 +++ public/bench/composer-arena/v0/methodology.md | 54 +++ .../composer-arena/v0/release.preview.json | 212 ++++++++++ .../composer-arena/v0/release.schema.json | 222 +++++++++++ scripts/check-composer-arena.mjs | 119 ++++++ scripts/next-font-google-mock.cjs | 11 + src/app/bench/composer-arena/arena-client.tsx | 364 ++++++++++++++++++ src/app/bench/composer-arena/data.ts | 35 ++ src/app/bench/composer-arena/page.tsx | 232 +++++++++++ src/app/bench/composer-arena/types.ts | 126 ++++++ src/app/bench/page.tsx | 25 ++ 15 files changed, 1528 insertions(+), 1 deletion(-) create mode 100644 public/bench/composer-arena/v0/battle.schema.json create mode 100644 public/bench/composer-arena/v0/battles.preview.jsonl create mode 100644 public/bench/composer-arena/v0/checksums.json create mode 100644 public/bench/composer-arena/v0/handoff.md create mode 100644 public/bench/composer-arena/v0/methodology.md create mode 100644 public/bench/composer-arena/v0/release.preview.json create mode 100644 public/bench/composer-arena/v0/release.schema.json create mode 100644 scripts/check-composer-arena.mjs create mode 100644 scripts/next-font-google-mock.cjs create mode 100644 src/app/bench/composer-arena/arena-client.tsx create mode 100644 src/app/bench/composer-arena/data.ts create mode 100644 src/app/bench/composer-arena/page.tsx create mode 100644 src/app/bench/composer-arena/types.ts diff --git a/package.json b/package.json index 96d7236..b0cdb8f 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "version": "0.1.0", "private": true, "scripts": { - "check:bench": "node scripts/check-bench-route.mjs", + "check:arena": "node scripts/check-composer-arena.mjs", + "check:bench": "node scripts/check-bench-route.mjs && node scripts/check-composer-arena.mjs", "dev": "next dev", "build": "next build", "start": "next start", diff --git a/public/bench/composer-arena/v0/battle.schema.json b/public/bench/composer-arena/v0/battle.schema.json new file mode 100644 index 0000000..d7b94fe --- /dev/null +++ b/public/bench/composer-arena/v0/battle.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/v0/battle.schema.json", + "title": "Composer Arena anonymous battle replay", + "type": "object", + "additionalProperties": false, + "required": ["battle_id", "track", "task_slice", "prompt", "source", "left", "right"], + "properties": { + "battle_id": { "type": "string", "minLength": 1 }, + "track": { "enum": ["controlled-agent", "end-to-end-system"] }, + "task_slice": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string", "minLength": 1 }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["fixture_id", "source_family_id", "revision", "rankable"], + "properties": { + "fixture_id": { "type": "string", "minLength": 1 }, + "source_family_id": { "type": "string", "minLength": 1 }, + "revision": { "type": "string", "minLength": 1 }, + "rankable": { "type": "boolean" } + } + }, + "left": { "$ref": "#/$defs/candidate" }, + "right": { "$ref": "#/$defs/candidate" } + }, + "$defs": { + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "model", + "provider", + "version", + "output_title", + "output_summary", + "evidence" + ], + "properties": { + "candidate_id": { "type": "string", "minLength": 1 }, + "model": { "type": "string", "minLength": 1 }, + "provider": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "output_title": { "type": "string", "minLength": 1 }, + "output_summary": { "type": "string", "minLength": 1 }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "href"], + "properties": { + "label": { "type": "string", "minLength": 1 }, + "href": { "type": "string", "pattern": "^/bench/composer-arena/v0/" } + } + } + } + } + } + } +} diff --git a/public/bench/composer-arena/v0/battles.preview.jsonl b/public/bench/composer-arena/v0/battles.preview.jsonl new file mode 100644 index 0000000..e0020ff --- /dev/null +++ b/public/bench/composer-arena/v0/battles.preview.jsonl @@ -0,0 +1,2 @@ +{"battle_id":"preview-controlled-001","track":"controlled-agent","task_slice":"temporal-ordering","prompt":"Create an evidence-grounded edit plan that opens on the constraint, preserves source chronology, and cites every selected window.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-chronology","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"sol-medium-v1","model":"GPT-5.6 Sol","provider":"AgenticFlow","version":"sol-medium-v1","output_title":"Output A","output_summary":"Three-window plan with explicit evidence IDs, chronological ordering, and a short unsupported transition note.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Methodology","href":"/bench/composer-arena/v0/methodology.md"}]},"right":{"candidate_id":"grok-4.5-medium-v1","model":"Grok 4.5","provider":"AgenticFlow","version":"grok-4.5-medium-v1","output_title":"Output B","output_summary":"Three-window plan with complete evidence IDs and valid ranges; the opening constraint is satisfied after a brief setup beat.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Release fixture","href":"/bench/composer-arena/v0/release.preview.json"}]}} +{"battle_id":"preview-system-001","track":"end-to-end-system","task_slice":"reviewable-delivery","prompt":"Produce a structurally valid 45-second review cut with synchronized source windows, no blocking gaps, and a provenance-complete delivery record.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-delivery","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"grok-4.5-medium-v1-composer-v0","model":"Grok 4.5 + Composer","provider":"AgenticFlow / Pixel ML","version":"grok-4.5-medium-v1 / composer-v0","output_title":"Output A","output_summary":"Valid 45-second timeline with complete provenance; one caption-safe-area warning remains non-blocking.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Schema","href":"/bench/composer-arena/v0/release.schema.json"}]},"right":{"candidate_id":"sol-medium-v1-composer-v0","model":"GPT-5.6 Sol + Composer","provider":"AgenticFlow / Pixel ML","version":"sol-medium-v1 / composer-v0","output_title":"Output B","output_summary":"Valid synchronized timeline and clean structural gate; delivery note omits the synthetic source-family label.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Handoff contract","href":"/bench/composer-arena/v0/handoff.md"}]}} diff --git a/public/bench/composer-arena/v0/checksums.json b/public/bench/composer-arena/v0/checksums.json new file mode 100644 index 0000000..59b1181 --- /dev/null +++ b/public/bench/composer-arena/v0/checksums.json @@ -0,0 +1,11 @@ +{ + "algorithm": "sha256", + "files": { + "release.schema.json": "a08a171fb29b3cbed0e173f549cf566597fb840168b34f06181e400d7fdd6ace", + "battle.schema.json": "b235183b07bf469172c79e8746275f2dd81dc30517667db5fb9a40f07c1f1572", + "release.preview.json": "bd4d755cc09154a5fed315299e8771908f3f956e2b63a8136b8544680dc01371", + "battles.preview.jsonl": "cb03fa3f39acc9c6488aa2beb45b08dc3cefe9fcaae236a66805873bd05d0d6c", + "methodology.md": "94ce70458152c1123a279181256c9a17d9321733c556317e54c65b2e85907917", + "handoff.md": "05524e739f8ce21e284ab83d4c54eba50b751b57f5b8ceaeeb0c05e95324305e" + } +} diff --git a/public/bench/composer-arena/v0/handoff.md b/public/bench/composer-arena/v0/handoff.md new file mode 100644 index 0000000..6c7d5f1 --- /dev/null +++ b/public/bench/composer-arena/v0/handoff.md @@ -0,0 +1,50 @@ +# Composer Arena real-data handoff + +The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. Real public artifacts can replace those files without changing React code when they preserve the schema and filenames below. + +## Required files + +1. `release.preview.json` — one release object validated against `release.schema.json`. +2. `battles.preview.jsonl` — one JSON object per replayable battle, with each line validated against `battle.schema.json`. +3. `checksums.json` — raw SHA-256 values for the schema, release, battles, methodology, and this handoff. +4. `methodology.md` — frozen methodology, estimator, vote policy, source/rights boundary, contamination policy, and limitations. + +Before a rankable release, change `release_mode` to `governed_rankable_release` and `official_ranking_enabled` to `true` only after the already-green FineVideo release marker is joined by real runner records, a governed vote backend when votes are included, a frozen statistical policy, and command-room review. Synthetic or local/demo data must remain `synthetic_non_rankable_preview`. + +## Release rows + +Every track row must provide: + +- `standing`: `ranked`, `tie`, or `insufficient_evidence`; +- `display_rank`: ordinal text, tie text such as `T1`, or `—`; +- model, provider, version, and evaluation date; +- Arena score, two-element 95% confidence interval, and battle count; +- deterministic task success and failure rate in `[0, 1]`; +- p50 latency in milliseconds; +- input, output, cache-read, and cache-write token totals; +- estimated USD cost per run; +- at least one per-task slice with task ID, label, success rate, and battle count. + +`controlled-agent` and `end-to-end-system` must remain separate track objects. Exploratory capability cards must use `status: non_rankable` and cannot be folded into Arena score. + +## Battle JSONL contract + +Each line must be a self-contained object with: + +- `battle_id`, `track`, `task_slice`, and the public-safe task prompt; +- a `source` object containing fixture/cell identity, source-family ID, immutable revision, and `rankable`; +- `left` and `right` candidates containing a stable candidate ID, hidden model/provider/version identity, public output title/summary, and one or more provenance/evidence links. + +The browser may swap left/right display order. Judgment choices are exactly `left`, `right`, `tie`, and `both_bad`. The public client reveals identity only after a judgment and sends no vote request. + +## Promotion gates + +- FineVideo repo and revision match the frozen contract. +- Raw contract and selection-release SHA-256 values match reviewed markers. +- Rankable runner/runtime records exist and validate; a green source-release gate alone is not a model-result release. +- Public-eval and private-test source families are disjoint. +- No hidden test data, customer/GMA data, credentials, signed URLs, or raw reasoning traces enter public artifacts. +- Every model route passes exact-provider/model/version preflight with no fallback substitution. +- The statistical exporter emits ties and insufficient-evidence states rather than forcing ranks. +- Official votes, if any, come only from a governed backend with a versioned inclusion policy. +- Schema validation, checksum verification, lint, build, route contract checks, and HTTP/UI smoke are green. diff --git a/public/bench/composer-arena/v0/methodology.md b/public/bench/composer-arena/v0/methodology.md new file mode 100644 index 0000000..121c41f --- /dev/null +++ b/public/bench/composer-arena/v0/methodology.md @@ -0,0 +1,54 @@ +# Composer Archive-to-Output Arena v0 methodology + +Status: **non-rankable preview**. The checked-in metrics and battle outputs are synthetic interface fixtures, not measured model results. + +## Tracks + +- `controlled-agent` compares evidence-grounded archive understanding, temporal reasoning, and executable edit planning over the same frozen evidence and brief. +- `end-to-end-system` compares complete Composer executions over the same released cell and brief. Timeline mutation must use the frozen Composer tool contract and pass structural gates. +- The two tracks are reported separately and never collapsed into one leaderboard. + +## Standing and uncertainty + +Arena score is a presentation field for the future governed pairwise estimator. A real release must document the estimator, seed, bootstrap procedure, tie threshold, and vote inclusion policy. The public surface shows the frozen 95% confidence interval and battle count beside every score. + +An ordinal standing is forbidden when the minimum battle count is not met or required evidence is incomplete. A tie is displayed when the governed comparison cannot establish separation at 95% confidence. Synthetic preview rows remain non-rankable regardless of their displayed fixture standing. + +Deterministic task success is computed independently from preference battles. Failure rate retains explicit terminal failures in the denominator. Latency, token totals, and estimated cost use frozen run records; they never disappear because a run failed. + +## Anonymous battle replay + +The replay visually hides model identity until the visitor records `A is better`, `B is better`, `Tie`, or `Both bad`. A/B order can be swapped before judgment. The judgment is stored only in React state for the current browser view: there is no API request, persistence, identity, anti-abuse control, or official ranking effect. Synthetic identities are present in the downloadable fixture, so preview reveal is not a secrecy boundary; a governed backend must withhold real identity payloads until judgment. + +Any future governed vote backend requires a reviewed sampling policy, replay identity, duplicate/abuse controls, immutable battle and output hashes, model reveal rules, audit export, deletion/privacy handling, and a versioned policy that says exactly which votes enter an official release. + +## FineVideo source and attribution + +The rankable source contract freezes `HuggingFaceFV/finevideo` at immutable revision `84c74091e1c6ee7a5dffabfafb5c9033e4718883`. The command-room-audited selection manifest contains 24 `public-eval` source families. Those selected cells carry the `CC-BY` claim from FineVideo publisher metadata and dataset terms, plus source-specific creator attribution text. + +Public examples must retain the exact per-cell attribution and provenance chain. This preview fetches or re-hosts no media and contains no real evaluated output. It does not broaden a per-cell rights record into a blanket claim about unrelated FineVideo content. + +Public-eval and private-test source families must remain disjoint. Hidden-test families, annotations, prompts, and failure history stay private and are never exposed through the public route or battle replay. + +## Frozen evidence + +- System-eval contract SHA-256: `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624` +- FineVideo selection release SHA-256: `d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926` +- FineVideo selection-manifest SHA-256: `39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4` + +The separate `FINEVIDEO_RELEASED` marker now exists and the official release check passes for 24 cells with the contract and release hashes above. That gate authorizes only the frozen source cells. The rankable runner/runtime has not yet produced real model records, so this public surface still publishes no measured score, output, or rank. + +## Contamination and privacy boundaries + +No evaluated model may judge its own or another baseline's output. Training overlap, prompt leakage, source-family reuse, hidden-test tuning, or model substitution disables official rankability. Human editorial review is blinded, uses two reviewers plus adjudication, and remains separate from deterministic scoring. + +Public exports may contain aggregate metrics, public-safe artifact hashes, required FineVideo attribution, and reviewed examples only. They must never contain credentials, signed URLs, provider headers, hidden annotations, raw reasoning traces, unnecessary personal metadata, customer data, or GMA data. + +## Known limitations + +- All preview scores, intervals, costs, and outputs are synthetic. +- The FineVideo source release is green, but the rankable runner/runtime is still being repaired and has produced no public model records. +- The public surface does not implement an official vote backend. +- The frozen system-eval contract currently uses one repetition per real cell; reliability claims need a reviewed repeated-run extension. +- Kimi K3 uses Composer's expressible `high` effort because the runtime cannot express the provider's published `max` effort. +- Editorial preference and long-horizon recovery remain exploratory, non-rankable capability views. diff --git a/public/bench/composer-arena/v0/release.preview.json b/public/bench/composer-arena/v0/release.preview.json new file mode 100644 index 0000000..3f274f3 --- /dev/null +++ b/public/bench/composer-arena/v0/release.preview.json @@ -0,0 +1,212 @@ +{ + "schema_version": 1, + "contract_version": "composer-arena-public-release-v0.1", + "release_id": "composer-arena-v0-synthetic-preview-2026-07-19", + "generated_at": "2026-07-19T12:00:00Z", + "release_mode": "synthetic_non_rankable_preview", + "official_ranking_enabled": false, + "official_vote_backend": "not_configured", + "display_label": "non-rankable preview", + "dataset": { + "repo_id": "HuggingFaceFV/finevideo", + "revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", + "license_claim": "CC-BY", + "license_scope": "The 24 command-room-audited public-eval cells carry CC-BY publisher metadata in the frozen selection manifest. Each real public example must retain its source-specific attribution text.", + "attribution": "FineVideo, HuggingFaceFV/finevideo at immutable revision 84c74091e1c6ee7a5dffabfafb5c9033e4718883. Source-specific creator attribution is retained per released cell.", + "contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", + "finevideo_release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + "selection_manifest_sha256": "39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4", + "public_eval_source_family_count": 24, + "split_policy": { + "source_family_disjoint": true, + "public_eval": "Only command-room-released source families may appear in public evaluation.", + "private_test": "Private-test families and annotations remain unpublished and must never appear in prompts or local demo artifacts.", + "hidden_test_published": false + } + }, + "evaluation_policy": { + "confidence_level": 0.95, + "minimum_battles_for_standing": 20, + "tie_rule": "Show a tie when the frozen comparison cannot establish a separated standing at 95% confidence.", + "insufficient_evidence_rule": "Do not assign an ordinal standing below the minimum battle count or when required evidence is incomplete.", + "local_vote_policy": "Replay judgments are browser-local demonstrations. They are not transmitted, persisted, or included in official standings.", + "contamination_limit": "No evaluated model may judge any baseline output. Training overlap, prompt leakage, source-family reuse, or hidden-test tuning disables official rankability.", + "privacy_limit": "Public exports contain aggregate metrics, public-safe hashes, required attribution, and reviewed examples only; never credentials, signed URLs, hidden annotations, raw reasoning traces, customer data, or GMA data." + }, + "tracks": [ + { + "id": "controlled-agent", + "title": "Controlled agent", + "description": "Same frozen archive evidence and task brief; scores evidence-grounded planning and temporal reasoning without running the full production system.", + "rows": [ + { + "standing": "tie", + "display_rank": "T1", + "model": "GPT-5.6 Sol", + "provider": "AgenticFlow", + "version": "sol-medium-v1", + "evaluation_date": "2026-07-19", + "arena_score": 1074, + "arena_score_ci95": [1051, 1096], + "battle_count": 48, + "deterministic_task_success": 0.833, + "failure_rate": 0.063, + "latency_ms_p50": 42100, + "tokens": { "input": 18400, "output": 6100, "cache_read": 9200, "cache_write": 0 }, + "estimated_cost_usd_per_run": 0.72, + "per_task_slices": [ + { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.875, "battle_count": 16 }, + { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.813, "battle_count": 16 }, + { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.813, "battle_count": 16 } + ] + }, + { + "standing": "tie", + "display_rank": "T1", + "model": "Grok 4.5", + "provider": "AgenticFlow", + "version": "grok-4.5-medium-v1", + "evaluation_date": "2026-07-19", + "arena_score": 1068, + "arena_score_ci95": [1046, 1090], + "battle_count": 48, + "deterministic_task_success": 0.813, + "failure_rate": 0.083, + "latency_ms_p50": 39700, + "tokens": { "input": 17600, "output": 6800, "cache_read": 7600, "cache_write": 0 }, + "estimated_cost_usd_per_run": 0.66, + "per_task_slices": [ + { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.813, "battle_count": 16 }, + { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.813, "battle_count": 16 }, + { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.813, "battle_count": 16 } + ] + }, + { + "standing": "insufficient_evidence", + "display_rank": "—", + "model": "Kimi K3", + "provider": "AgenticFlow", + "version": "kimi-k3-high-v1", + "evaluation_date": "2026-07-19", + "arena_score": 1059, + "arena_score_ci95": [982, 1131], + "battle_count": 8, + "deterministic_task_success": 0.75, + "failure_rate": 0.125, + "latency_ms_p50": 51400, + "tokens": { "input": 20100, "output": 7200, "cache_read": 8100, "cache_write": 0 }, + "estimated_cost_usd_per_run": 0.81, + "per_task_slices": [ + { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.75, "battle_count": 4 }, + { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.75, "battle_count": 2 }, + { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.75, "battle_count": 2 } + ] + } + ] + }, + { + "id": "end-to-end-system", + "title": "End-to-end system", + "description": "Same released cells and brief, executed through Composer tooling and structural gates; never blended with the controlled-agent table.", + "rows": [ + { + "standing": "tie", + "display_rank": "T1", + "model": "GPT-5.6 Sol + Composer", + "provider": "AgenticFlow / Pixel ML", + "version": "sol-medium-v1 / composer-v0", + "evaluation_date": "2026-07-19", + "arena_score": 1042, + "arena_score_ci95": [1014, 1071], + "battle_count": 36, + "deterministic_task_success": 0.778, + "failure_rate": 0.111, + "latency_ms_p50": 118400, + "tokens": { "input": 26300, "output": 8700, "cache_read": 11200, "cache_write": 0 }, + "estimated_cost_usd_per_run": 1.18, + "per_task_slices": [ + { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.833, "battle_count": 12 }, + { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 0.833, "battle_count": 12 }, + { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.667, "battle_count": 12 } + ] + }, + { + "standing": "tie", + "display_rank": "T1", + "model": "Grok 4.5 + Composer", + "provider": "AgenticFlow / Pixel ML", + "version": "grok-4.5-medium-v1 / composer-v0", + "evaluation_date": "2026-07-19", + "arena_score": 1036, + "arena_score_ci95": [1008, 1065], + "battle_count": 36, + "deterministic_task_success": 0.75, + "failure_rate": 0.139, + "latency_ms_p50": 109600, + "tokens": { "input": 24900, "output": 9100, "cache_read": 9800, "cache_write": 0 }, + "estimated_cost_usd_per_run": 1.09, + "per_task_slices": [ + { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.75, "battle_count": 12 }, + { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 0.833, "battle_count": 12 }, + { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.667, "battle_count": 12 } + ] + }, + { + "standing": "insufficient_evidence", + "display_rank": "—", + "model": "GLM 5.2 + Composer", + "provider": "AgenticFlow / Pixel ML", + "version": "glm-5.2-high-v1 / composer-v0", + "evaluation_date": "2026-07-19", + "arena_score": 1028, + "arena_score_ci95": [946, 1112], + "battle_count": 6, + "deterministic_task_success": 0.667, + "failure_rate": 0.167, + "latency_ms_p50": 126800, + "tokens": { "input": 23100, "output": 7900, "cache_read": 7400, "cache_write": 0 }, + "estimated_cost_usd_per_run": 0.94, + "per_task_slices": [ + { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.667, "battle_count": 2 }, + { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 1, "battle_count": 2 }, + { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.333, "battle_count": 2 } + ] + } + ] + } + ], + "exploratory_capabilities": [ + { + "id": "direct-perception", + "title": "Direct visual perception", + "status": "non_rankable", + "summary": "Can the route inspect released frames directly rather than reason only over frozen derived evidence?", + "evidence_scope": "Capability declaration and preflight only; no synthetic score is converted into a rank.", + "limitation": "GLM 5.2 is marked not-applicable for direct perception in the frozen contract, which is not treated as a failure." + }, + { + "id": "editorial-preference", + "title": "Editorial preference", + "status": "non_rankable", + "summary": "Blind human review of coherence, opening strength, cut naturalness, and caption readability.", + "evidence_scope": "Two reviewers plus adjudication are required for any real public editorial claim.", + "limitation": "Human preference remains separate from deterministic task success and the synthetic Arena score." + }, + { + "id": "long-horizon-recovery", + "title": "Long-horizon recovery", + "status": "non_rankable", + "summary": "Recovery from tool failures, stale state, and interrupted editing turns.", + "evidence_scope": "Exploratory fault-injection notes only.", + "limitation": "The v0 public contract does not yet freeze a comparable failure-injection distribution." + } + ], + "known_limitations": [ + "All scores, confidence intervals, battle counts, costs, and outputs in this preview are synthetic interface fixtures and must not be interpreted as measured model performance.", + "The FineVideo selection and FINEVIDEO_RELEASED marker pass the 24-cell release gate, but the rankable runner/runtime has not produced real model records; every displayed row remains synthetic and non-rankable.", + "One repetition per real cell is currently frozen in the system-eval contract; reliability claims require a reviewed repeated-run extension.", + "Kimi K3 is evaluated at Composer's expressible high effort because the runtime cannot express the provider's published max effort.", + "Local replay judgments are browser-only and have no governed identity, anti-abuse, sampling, or persistence backend.", + "Preview identity reveal is a visual interaction, not a secrecy boundary: synthetic candidate identities ship in the downloadable fixture. A governed backend must withhold real identities until judgment." + ] +} diff --git a/public/bench/composer-arena/v0/release.schema.json b/public/bench/composer-arena/v0/release.schema.json new file mode 100644 index 0000000..300f3b2 --- /dev/null +++ b/public/bench/composer-arena/v0/release.schema.json @@ -0,0 +1,222 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/v0/release.schema.json", + "title": "Composer Archive-to-Output Arena public release", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "contract_version", + "release_id", + "generated_at", + "release_mode", + "official_ranking_enabled", + "official_vote_backend", + "display_label", + "dataset", + "evaluation_policy", + "tracks", + "exploratory_capabilities", + "known_limitations" + ], + "properties": { + "schema_version": { "const": 1 }, + "contract_version": { "const": "composer-arena-public-release-v0.1" }, + "release_id": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "release_mode": { + "enum": ["synthetic_non_rankable_preview", "governed_rankable_release"] + }, + "official_ranking_enabled": { "type": "boolean" }, + "official_vote_backend": { "enum": ["not_configured", "governed_v1"] }, + "display_label": { "type": "string", "minLength": 1 }, + "dataset": { "$ref": "#/$defs/dataset" }, + "evaluation_policy": { "$ref": "#/$defs/evaluationPolicy" }, + "tracks": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { "$ref": "#/$defs/track" } + }, + "exploratory_capabilities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/exploratoryCapability" } + }, + "known_limitations": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "dataset": { + "type": "object", + "additionalProperties": false, + "required": [ + "repo_id", + "revision", + "license_claim", + "license_scope", + "attribution", + "contract_sha256", + "finevideo_release_sha256", + "selection_manifest_sha256", + "public_eval_source_family_count", + "split_policy" + ], + "properties": { + "repo_id": { "const": "HuggingFaceFV/finevideo" }, + "revision": { + "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" + }, + "license_claim": { "const": "CC-BY" }, + "license_scope": { "type": "string", "minLength": 1 }, + "attribution": { "type": "string", "minLength": 1 }, + "contract_sha256": { "$ref": "#/$defs/sha256" }, + "finevideo_release_sha256": { "$ref": "#/$defs/sha256" }, + "selection_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "public_eval_source_family_count": { "type": "integer", "minimum": 1 }, + "split_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_family_disjoint", + "public_eval", + "private_test", + "hidden_test_published" + ], + "properties": { + "source_family_disjoint": { "const": true }, + "public_eval": { "type": "string", "minLength": 1 }, + "private_test": { "type": "string", "minLength": 1 }, + "hidden_test_published": { "const": false } + } + } + } + }, + "evaluationPolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "confidence_level", + "minimum_battles_for_standing", + "tie_rule", + "insufficient_evidence_rule", + "local_vote_policy", + "contamination_limit", + "privacy_limit" + ], + "properties": { + "confidence_level": { "const": 0.95 }, + "minimum_battles_for_standing": { "type": "integer", "minimum": 1 }, + "tie_rule": { "type": "string", "minLength": 1 }, + "insufficient_evidence_rule": { "type": "string", "minLength": 1 }, + "local_vote_policy": { "type": "string", "minLength": 1 }, + "contamination_limit": { "type": "string", "minLength": 1 }, + "privacy_limit": { "type": "string", "minLength": 1 } + } + }, + "track": { + "type": "object", + "additionalProperties": false, + "required": ["id", "title", "description", "rows"], + "properties": { + "id": { "enum": ["controlled-agent", "end-to-end-system"] }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "rows": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/row" } + } + } + }, + "row": { + "type": "object", + "additionalProperties": false, + "required": [ + "standing", + "display_rank", + "model", + "provider", + "version", + "evaluation_date", + "arena_score", + "arena_score_ci95", + "battle_count", + "deterministic_task_success", + "failure_rate", + "latency_ms_p50", + "tokens", + "estimated_cost_usd_per_run", + "per_task_slices" + ], + "properties": { + "standing": { "enum": ["ranked", "tie", "insufficient_evidence"] }, + "display_rank": { "type": "string", "minLength": 1 }, + "model": { "type": "string", "minLength": 1 }, + "provider": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "evaluation_date": { "type": "string", "format": "date" }, + "arena_score": { "type": "number" }, + "arena_score_ci95": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "prefixItems": [{ "type": "number" }, { "type": "number" }] + }, + "battle_count": { "type": "integer", "minimum": 0 }, + "deterministic_task_success": { "type": "number", "minimum": 0, "maximum": 1 }, + "failure_rate": { "type": "number", "minimum": 0, "maximum": 1 }, + "latency_ms_p50": { "type": "number", "minimum": 0 }, + "tokens": { "$ref": "#/$defs/tokens" }, + "estimated_cost_usd_per_run": { "type": "number", "minimum": 0 }, + "per_task_slices": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/taskSlice" } + } + } + }, + "tokens": { + "type": "object", + "additionalProperties": false, + "required": ["input", "output", "cache_read", "cache_write"], + "properties": { + "input": { "type": "integer", "minimum": 0 }, + "output": { "type": "integer", "minimum": 0 }, + "cache_read": { "type": "integer", "minimum": 0 }, + "cache_write": { "type": "integer", "minimum": 0 } + } + }, + "taskSlice": { + "type": "object", + "additionalProperties": false, + "required": ["task_id", "label", "success_rate", "battle_count"], + "properties": { + "task_id": { "type": "string", "minLength": 1 }, + "label": { "type": "string", "minLength": 1 }, + "success_rate": { "type": "number", "minimum": 0, "maximum": 1 }, + "battle_count": { "type": "integer", "minimum": 0 } + } + }, + "exploratoryCapability": { + "type": "object", + "additionalProperties": false, + "required": ["id", "title", "status", "summary", "evidence_scope", "limitation"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "status": { "const": "non_rankable" }, + "summary": { "type": "string", "minLength": 1 }, + "evidence_scope": { "type": "string", "minLength": 1 }, + "limitation": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/scripts/check-composer-arena.mjs b/scripts/check-composer-arena.mjs new file mode 100644 index 0000000..929c15c --- /dev/null +++ b/scripts/check-composer-arena.mjs @@ -0,0 +1,119 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const root = resolve(import.meta.dirname, ".."); +const routeRoot = resolve(root, "src/app/bench/composer-arena"); +const artifactRoot = resolve(root, "public/bench/composer-arena/v0"); +const release = JSON.parse(readFileSync(resolve(artifactRoot, "release.preview.json"), "utf8")); +const schema = JSON.parse(readFileSync(resolve(artifactRoot, "release.schema.json"), "utf8")); +const checksums = JSON.parse(readFileSync(resolve(artifactRoot, "checksums.json"), "utf8")); +const battles = readFileSync(resolve(artifactRoot, "battles.preview.jsonl"), "utf8") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line)); +const page = readFileSync(resolve(routeRoot, "page.tsx"), "utf8"); +const client = readFileSync(resolve(routeRoot, "arena-client.tsx"), "utf8"); +const data = readFileSync(resolve(routeRoot, "data.ts"), "utf8"); +const methodology = readFileSync(resolve(artifactRoot, "methodology.md"), "utf8"); +const handoff = readFileSync(resolve(artifactRoot, "handoff.md"), "utf8"); + +function requireCondition(condition, message) { + if (!condition) throw new Error(message); +} + +function sha256(filename) { + return createHash("sha256").update(readFileSync(resolve(artifactRoot, filename))).digest("hex"); +} + +const requiredArtifacts = [ + "release.schema.json", + "battle.schema.json", + "release.preview.json", + "battles.preview.jsonl", + "checksums.json", + "methodology.md", + "handoff.md", +]; + +for (const artifact of requiredArtifacts) { + requireCondition(existsSync(resolve(artifactRoot, artifact)), `missing Composer Arena artifact: ${artifact}`); + requireCondition(page.includes(artifact), `Composer Arena page does not link ${artifact}`); +} +requireCondition(page.includes('const ARTIFACT_ROOT = "/bench/composer-arena/v0"'), "Composer Arena artifact root drifted"); + +for (const [filename, expected] of Object.entries(checksums.files)) { + requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid checksum for ${filename}`); + requireCondition(sha256(filename) === expected, `checksum mismatch for ${filename}`); +} + +requireCondition(schema.$id === "https://agentic.video/bench/composer-arena/v0/release.schema.json", "schema ID drifted"); +requireCondition(schema.additionalProperties === false, "release schema must fail closed on unknown top-level fields"); +requireCondition(release.release_mode === "synthetic_non_rankable_preview", "fixtures must remain a synthetic preview"); +requireCondition(release.official_ranking_enabled === false, "official ranking must remain disabled for fixtures"); +requireCondition(release.official_vote_backend === "not_configured", "preview must not imply a governed vote backend"); +requireCondition(release.display_label === "non-rankable preview", "preview label drifted"); +requireCondition(release.dataset.repo_id === "HuggingFaceFV/finevideo", "FineVideo repo drifted"); +requireCondition(release.dataset.revision === "84c74091e1c6ee7a5dffabfafb5c9033e4718883", "FineVideo revision drifted"); +requireCondition(release.dataset.license_claim === "CC-BY", "FineVideo licence claim drifted"); +requireCondition(release.dataset.contract_sha256 === "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", "contract hash drifted"); +requireCondition(release.dataset.finevideo_release_sha256 === "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", "FineVideo release hash drifted"); +requireCondition(release.dataset.split_policy.source_family_disjoint === true, "source families must remain disjoint"); +requireCondition(release.dataset.split_policy.hidden_test_published === false, "hidden test must remain unpublished"); + +const trackIds = release.tracks.map((track) => track.id).sort(); +requireCondition( + JSON.stringify(trackIds) === JSON.stringify(["controlled-agent", "end-to-end-system"]), + "release must contain exactly the two separate Arena tracks", +); + +for (const track of release.tracks) { + requireCondition(track.rows.length > 0, `${track.id} has no rows`); + requireCondition(track.rows.some((row) => row.standing === "tie"), `${track.id} must show a visible tie`); + requireCondition( + track.rows.some((row) => row.standing === "insufficient_evidence"), + `${track.id} must show insufficient evidence`, + ); + + for (const row of track.rows) { + requireCondition(row.model && row.provider && row.version && row.evaluation_date, `${track.id} row identity is incomplete`); + requireCondition(Number.isFinite(row.arena_score), `${track.id} row is missing Arena score`); + requireCondition(Array.isArray(row.arena_score_ci95) && row.arena_score_ci95.length === 2, `${track.id} row CI is invalid`); + requireCondition(Number.isInteger(row.battle_count) && row.battle_count >= 0, `${track.id} battle count is invalid`); + requireCondition(row.deterministic_task_success >= 0 && row.deterministic_task_success <= 1, `${track.id} task success is invalid`); + requireCondition(row.failure_rate >= 0 && row.failure_rate <= 1, `${track.id} failure rate is invalid`); + requireCondition(row.latency_ms_p50 >= 0, `${track.id} latency is invalid`); + requireCondition(Object.values(row.tokens).every((value) => Number.isInteger(value) && value >= 0), `${track.id} token usage is invalid`); + requireCondition(row.estimated_cost_usd_per_run >= 0, `${track.id} cost is invalid`); + requireCondition(row.per_task_slices.length > 0, `${track.id} per-task slices are missing`); + } +} + +requireCondition( + release.exploratory_capabilities.every((capability) => capability.status === "non_rankable"), + "exploratory capabilities must remain non-rankable", +); +requireCondition(battles.length >= 2, "preview needs controlled-agent and end-to-end battle replays"); +requireCondition(new Set(battles.map((battle) => battle.track)).size === 2, "battle fixtures must cover both tracks"); +for (const battle of battles) { + requireCondition(battle.source.rankable === false, `${battle.battle_id} must remain non-rankable`); + requireCondition(battle.left.evidence.length > 0 && battle.right.evidence.length > 0, `${battle.battle_id} lacks evidence links`); +} + +requireCondition(client.includes("Swap A/B order"), "battle replay must support swapped order"); +requireCondition(client.includes("Both bad"), "battle replay must support both-bad judgments"); +requireCondition(client.includes("Tie"), "battle replay must support tie judgments"); +requireCondition(client.includes("revealed={judgment !== null}"), "model identity must reveal only after judgment"); +requireCondition(client.includes("No request is sent"), "local vote exclusion must be visible"); +requireCondition(data.includes("release.preview.json") && data.includes("battles.preview.jsonl"), "route must be artifact-driven"); +requireCondition(methodology.includes("source families must remain disjoint"), "methodology is missing the source-family split"); +requireCondition(methodology.includes("CC-BY"), "methodology is missing FineVideo attribution scope"); +requireCondition(methodology.includes("official release check passes for 24 cells"), "methodology must bind the green source release"); +requireCondition(methodology.includes("runner/runtime"), "methodology must disclose the missing real runner records"); +requireCondition(handoff.includes("without changing React code"), "handoff must preserve schema-driven replacement"); +requireCondition(handoff.includes("battle.schema.json"), "handoff must bind JSONL rows to the battle schema"); +requireCondition(handoff.includes("governed vote backend"), "handoff is missing official vote governance"); +requireCondition(!page.includes("customer data") || page.includes("never"), "page must not normalize customer data use"); + +console.log("Composer Arena route contract check passed"); diff --git a/scripts/next-font-google-mock.cjs b/scripts/next-font-google-mock.cjs new file mode 100644 index 0000000..8a4ea99 --- /dev/null +++ b/scripts/next-font-google-mock.cjs @@ -0,0 +1,11 @@ +module.exports = { + "https://fonts.googleapis.com/css2?family=Chivo+Mono:wght@100..900&display=swap": ` + @font-face { + font-family: 'Chivo Mono'; + font-style: normal; + font-weight: 100 900; + font-display: swap; + src: url(mock-chivo-mono.woff2) format('woff2'); + } + `, +}; diff --git a/src/app/bench/composer-arena/arena-client.tsx b/src/app/bench/composer-arena/arena-client.tsx new file mode 100644 index 0000000..608986e --- /dev/null +++ b/src/app/bench/composer-arena/arena-client.tsx @@ -0,0 +1,364 @@ +"use client"; + +import { useMemo, useState } from "react"; + +import type { + ArenaBattle, + ArenaRelease, + ArenaTrackId, + BattleCandidate, +} from "./types"; + +type Judgment = "left" | "right" | "tie" | "both_bad"; + +const percent = new Intl.NumberFormat("en-US", { + style: "percent", + maximumFractionDigits: 1, +}); + +const integer = new Intl.NumberFormat("en-US"); + +function StandingBadge({ standing }: { standing: "ranked" | "tie" | "insufficient_evidence" }) { + if (standing === "insufficient_evidence") { + return ( + + insufficient evidence + + ); + } + + if (standing === "tie") { + return ( + + tie + + ); + } + + return ( + + ranked + + ); +} + +function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: ArenaTrackId }) { + const track = release.tracks.find((candidate) => candidate.id === trackId); + + if (!track) return null; + + return ( +
+
+

{track.title}

+

{track.description}

+
+ +
+ + + + + {[ + "Standing", + "Model / route", + "Arena score (95% CI)", + "Battles", + "Task success", + "Failure rate", + "p50 latency", + "Tokens", + "Est. cost / run", + "Per-task slices", + ].map((heading) => ( + + ))} + + + + {track.rows.map((row) => ( + + + + + + + + + + + + + ))} + +
{track.title} synthetic preview standings
+ {heading} +
+
{row.display_rank}
+
+ 		+
{row.model}
+
+ {row.provider}
{row.version}
{row.evaluation_date} +
+ 		+
{integer.format(row.arena_score)}
+
+ {integer.format(row.arena_score_ci95[0])}–{integer.format(row.arena_score_ci95[1])} +
+ 		{integer.format(row.battle_count)}{percent.format(row.deterministic_task_success)}{percent.format(row.failure_rate)}{(row.latency_ms_p50 / 1000).toFixed(1)}s + {integer.format(row.tokens.input)} in
+ {integer.format(row.tokens.output)} out
+ {integer.format(row.tokens.cache_read)} cache read + 		${row.estimated_cost_usd_per_run.toFixed(2)} +
    + {row.per_task_slices.map((slice) => ( +
  • +
    + {slice.label} + {percent.format(slice.success_rate)} +
    +
    • + ))} +
+
+
+
+ ); +} + +function CandidateCard({ + label, + candidate, + revealed, +}: { + label: "A" | "B"; + candidate: BattleCandidate; + revealed: boolean; +}) { + return ( +
+
+ + + {revealed ? candidate.model : "identity hidden"} + +
+

Output {label}

+

{candidate.output_summary}

+ + {revealed && ( +
+
+
+
Provider
+
{candidate.provider}
+
+
+
Version
+
{candidate.version}
+
+
+
+ {candidate.evidence.map((link) => ( + + {link.label} + + ))} +
+
+ )} +
+ ); +} + +function BattleReplay({ battles }: { battles: ArenaBattle[] }) { + const [battleIndex, setBattleIndex] = useState(0); + const [swapped, setSwapped] = useState(false); + const [judgment, setJudgment] = useState(null); + const battle = battles[battleIndex]; + + const [candidateA, candidateB] = useMemo( + () => (swapped ? [battle.right, battle.left] : [battle.left, battle.right]), + [battle, swapped], + ); + + const chooseBattle = (index: number) => { + setBattleIndex(index); + setSwapped(false); + setJudgment(null); + }; + + const swap = () => { + setSwapped((value) => !value); + setJudgment(null); + }; + + return ( +
+
+
+ + +
+ +
+ +
+
+ {battle.track}{battle.task_slice} +
+

{battle.prompt}

+

+ Synthetic source: {battle.source.fixture_id} · {battle.source.source_family_id} · non-rankable +

+
+ +
+ + +
+ +
+ Record a local demo judgment +

+ This selection stays in this page only. No request is sent and official standings never change. +

+
+ {([ + ["left", "A is better"], + ["tie", "Tie"], + ["right", "B is better"], + ["both_bad", "Both bad"], + ] as const).map(([value, label]) => ( + + ))} +
+
+ +
+ {judgment ? ( +

+ Identity revealed. Demo judgment recorded locally as {judgment.replace("_", " ")}; excluded from official ranking. +

+ ) : ( +

Choose only after comparing the anonymous outputs.

+ )} +
+
+ ); +} + +export default function ArenaClient({ release, battles }: { release: ArenaRelease; battles: ArenaBattle[] }) { + const [trackId, setTrackId] = useState("controlled-agent"); + + return ( + <> +
+
+
+
+

Preview standings

+

Separate views; no forced ranks

+
+
+ {release.tracks.map((track) => ( + + ))} +
+
+ +
+ Synthetic fixture values only. Ties are explicit; rows below {release.evaluation_policy.minimum_battles_for_standing} governed battles show insufficient evidence instead of an ordinal rank. +
+ + +
+
+ +
+
+

Exploratory capabilities

+

Observed separately; never converted into rank

+
+ {release.exploratory_capabilities.map((capability) => ( +
+ + non-rankable + +

{capability.title}

+

{capability.summary}

+

+ Evidence: {capability.evidence_scope} +

+

+ Limit: {capability.limitation} +

+
+ ))} +
+
+
+ +
+
+

Anonymous A/B replay

+

Judge first; reveal identity second

+

+ Swap order to test position sensitivity, choose a winner, tie, or both-bad state, then inspect model identity and evidence links. +

+
+
+
+ + ); +} diff --git a/src/app/bench/composer-arena/data.ts b/src/app/bench/composer-arena/data.ts new file mode 100644 index 0000000..d06d00e --- /dev/null +++ b/src/app/bench/composer-arena/data.ts @@ -0,0 +1,35 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import type { ArenaBattle, ArenaChecksums, ArenaRelease } from "./types"; + +const artifactRoot = path.join( + process.cwd(), + "public", + "bench", + "composer-arena", + "v0", +); + +async function readJson(filename: string): Promise { + return JSON.parse(await readFile(path.join(artifactRoot, filename), "utf8")) as T; +} + +async function readJsonLines(filename: string): Promise { + const raw = await readFile(path.join(artifactRoot, filename), "utf8"); + return raw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as T); +} + +export async function loadComposerArena() { + const [release, battles, checksums] = await Promise.all([ + readJson("release.preview.json"), + readJsonLines("battles.preview.jsonl"), + readJson("checksums.json"), + ]); + + return { release, battles, checksums }; +} diff --git a/src/app/bench/composer-arena/page.tsx b/src/app/bench/composer-arena/page.tsx new file mode 100644 index 0000000..5a55164 --- /dev/null +++ b/src/app/bench/composer-arena/page.tsx @@ -0,0 +1,232 @@ +import type { Metadata } from "next"; +import Link from "next/link"; + +import ArenaClient from "./arena-client"; +import { loadComposerArena } from "./data"; + +const ARTIFACT_ROOT = "/bench/composer-arena/v0"; + +export const metadata: Metadata = { + title: "Composer Archive-to-Output Arena v0 — Agentic Video Benchmarks", + description: + "Non-rankable, synthetic public preview of the Composer Archive-to-Output Arena v0 contract, metrics, anonymous battle replay, and real-data handoff.", + alternates: { canonical: "/bench/composer-arena" }, + openGraph: { + title: "Composer Archive-to-Output Arena v0", + description: "Synthetic non-rankable preview with separate controlled-agent and end-to-end system views.", + type: "website", + url: "https://agentic.video/bench/composer-arena", + }, +}; + +function SectionHeading({ eyebrow, children }: { eyebrow: string; children: React.ReactNode }) { + return ( +
+

{eyebrow}

+

{children}

+
+ ); +} + +function HashValue({ children }: { children: string }) { + return {children}; +} + +export default async function ComposerArenaPage() { + const { release, battles, checksums } = await loadComposerArena(); + + const artifactLinks = [ + ["Release fixture (JSON)", `${ARTIFACT_ROOT}/release.preview.json`], + ["Battle replay fixtures (JSONL)", `${ARTIFACT_ROOT}/battles.preview.jsonl`], + ["Release schema (JSON Schema)", `${ARTIFACT_ROOT}/release.schema.json`], + ["Battle schema (JSON Schema)", `${ARTIFACT_ROOT}/battle.schema.json`], + ["Checksums", `${ARTIFACT_ROOT}/checksums.json`], + ["Methodology", `${ARTIFACT_ROOT}/methodology.md`], + ["Real-data handoff", `${ARTIFACT_ROOT}/handoff.md`], + ] as const; + + return ( +
+
+
+
+ av + + Benchmarks + +
+ + Composer Arena by Pixel ML + +
+
+ +
+
+
+ {release.display_label} +
+

+ Agentic Video Benchmarks by Pixel ML +

+

+ Composer Archive-to-Output Arena v0 +

+

+ A publication-candidate surface for comparing evidence-grounded editing agents and complete Composer systems. Every visible result is a synthetic UI fixture until governed real outputs replace the artifacts below. +

+ +
+ Official ranking is disabled. No model or media call was made for this page, and local replay judgments are excluded from all standings. +
+ +
+ {[ + ["Release", release.release_id], + ["Mode", release.release_mode], + ["Generated", release.generated_at], + ["Vote backend", release.official_vote_backend], + ].map(([term, value]) => ( +
+
{term}
+
{value}
+
+ ))} +
+
+
+ + + +
+
+ Deterministic evidence first; preference stays bounded +
+ {[ + [ + "Standing policy", + `${release.evaluation_policy.tie_rule} ${release.evaluation_policy.insufficient_evidence_rule}`, + ], + [ + "Vote boundary", + release.evaluation_policy.local_vote_policy, + ], + [ + "Contamination gate", + release.evaluation_policy.contamination_limit, + ], + ].map(([title, body]) => ( +
+

{title}

+

{body}

+
+ ))} +
+
+

Privacy and publication boundary

+

+ {release.evaluation_policy.privacy_limit} +

+
+
+
+ +
+
+
+ Immutable revision and bounded CC-BY claim +

+ {release.dataset.license_scope} +

+

+ Attribution: {release.dataset.attribution} +

+ + View the immutable FineVideo dataset revision + +
+ +
+ Public evidence is not hidden-test evidence +
+
+
Audited public-eval families
+
{release.dataset.public_eval_source_family_count}
+
+
+
Hidden test published
+
No
+
+
+
    +
  • Public eval: {release.dataset.split_policy.public_eval}
    • +
  • Private test: {release.dataset.split_policy.private_test}
    • +
  • Disjoint: the same source family cannot cross the split.
    • +
+
+
+
+ +
+
+ Contract, source release, and public artifact hashes +
+
+

System-eval contract SHA-256

+ {release.dataset.contract_sha256} +
+
+

FineVideo release SHA-256

+ {release.dataset.finevideo_release_sha256} +
+
+

Public preview release SHA-256

+ {checksums.files["release.preview.json"]} +
+
+

+ The command-room FineVideo selection and release marker pass for 24 cells. The rankable runner/runtime has not produced real model records, so these hashes establish source provenance only—not measured performance or an official ranking. +

+
+
+ +
+
+ What this preview does not prove +
    + {release.known_limitations.map((limitation, index) => ( +
  1. + {String(index + 1).padStart(2, "0")} + {limitation} +
    2. + ))} +
+
+
+ +
+
+ Downloadable, replaceable public artifacts +

+ The route reads the JSON and JSONL artifacts at build time. A governed real release replaces the files under the same schema and filenames, so the public UI does not require model-specific code changes. +

+
+ {artifactLinks.map(([label, href]) => ( + + {label} + + ))} +
+
+
+
+ ); +} diff --git a/src/app/bench/composer-arena/types.ts b/src/app/bench/composer-arena/types.ts new file mode 100644 index 0000000..3a5ceeb --- /dev/null +++ b/src/app/bench/composer-arena/types.ts @@ -0,0 +1,126 @@ +export type ArenaTrackId = "controlled-agent" | "end-to-end-system"; + +export type ArenaStanding = "ranked" | "tie" | "insufficient_evidence"; + +export type TokenUsage = { + input: number; + output: number; + cache_read: number; + cache_write: number; +}; + +export type TaskSlice = { + task_id: string; + label: string; + success_rate: number; + battle_count: number; +}; + +export type ArenaRow = { + standing: ArenaStanding; + display_rank: string; + model: string; + provider: string; + version: string; + evaluation_date: string; + arena_score: number; + arena_score_ci95: [number, number]; + battle_count: number; + deterministic_task_success: number; + failure_rate: number; + latency_ms_p50: number; + tokens: TokenUsage; + estimated_cost_usd_per_run: number; + per_task_slices: TaskSlice[]; +}; + +export type ArenaTrack = { + id: ArenaTrackId; + title: string; + description: string; + rows: ArenaRow[]; +}; + +export type ExploratoryCapability = { + id: string; + title: string; + status: "non_rankable"; + summary: string; + evidence_scope: string; + limitation: string; +}; + +export type ArenaRelease = { + schema_version: 1; + contract_version: "composer-arena-public-release-v0.1"; + release_id: string; + generated_at: string; + release_mode: "synthetic_non_rankable_preview" | "governed_rankable_release"; + official_ranking_enabled: boolean; + official_vote_backend: "not_configured" | "governed_v1"; + display_label: string; + dataset: { + repo_id: string; + revision: string; + license_claim: string; + license_scope: string; + attribution: string; + contract_sha256: string; + finevideo_release_sha256: string; + selection_manifest_sha256: string; + public_eval_source_family_count: number; + split_policy: { + source_family_disjoint: true; + public_eval: string; + private_test: string; + hidden_test_published: false; + }; + }; + evaluation_policy: { + confidence_level: 0.95; + minimum_battles_for_standing: number; + tie_rule: string; + insufficient_evidence_rule: string; + local_vote_policy: string; + contamination_limit: string; + privacy_limit: string; + }; + tracks: ArenaTrack[]; + exploratory_capabilities: ExploratoryCapability[]; + known_limitations: string[]; +}; + +export type EvidenceLink = { + label: string; + href: string; +}; + +export type BattleCandidate = { + candidate_id: string; + model: string; + provider: string; + version: string; + output_title: string; + output_summary: string; + evidence: EvidenceLink[]; +}; + +export type ArenaBattle = { + battle_id: string; + track: ArenaTrackId; + task_slice: string; + prompt: string; + source: { + fixture_id: string; + source_family_id: string; + revision: string; + rankable: boolean; + }; + left: BattleCandidate; + right: BattleCandidate; +}; + +export type ArenaChecksums = { + algorithm: "sha256"; + files: Record; +}; diff --git a/src/app/bench/page.tsx b/src/app/bench/page.tsx index 1097bfd..3343b2a 100644 --- a/src/app/bench/page.tsx +++ b/src/app/bench/page.tsx @@ -124,6 +124,31 @@ export default function BenchPage() { +
+
+ Composer Archive-to-Output Arena v0 +
+
+
+ + non-rankable preview + +

Archive understanding to reviewable output

+

+ Synthetic, schema-driven standings, track-separated metrics, non-rankable capability cards, and anonymous A/B replay. No official vote backend or measured model result is active. +

+
+ + Open Composer Arena + +
+
+
+
+
Contracts live; real execution stopped From a0300b9a615c47cbc28610b72226d49ee4156f1e Mon Sep 17 00:00:00 2001 From: Sean Phan Date: Sun, 19 Jul 2026 05:43:59 -0700 Subject: [PATCH 2/4] feat: publish Composer Phase 1 pilot evidence --- package-lock.json | 96 +++++++- package.json | 1 + .../evidence/pilot/2026-07-19/README.md | 17 ++ .../evidence/pilot/2026-07-19/checksums.json | 8 + .../pilot/2026-07-19/phase-1-results.json | 161 +++++++++++++ .../2026-07-19/phase-1-results.schema.json | 165 +++++++++++++ public/bench/composer-arena/v0/checksums.json | 8 +- public/bench/composer-arena/v0/handoff.md | 13 +- public/bench/composer-arena/v0/methodology.md | 15 +- .../composer-arena/v0/release.preview.json | 77 +++--- .../composer-arena/v0/release.schema.json | 62 ++++- scripts/check-composer-arena.mjs | 84 ++++++- src/app/bench/composer-arena/arena-client.tsx | 11 +- .../evidence/pilot/2026-07-19/data.ts | 116 +++++++++ .../evidence/pilot/2026-07-19/page.tsx | 227 ++++++++++++++++++ src/app/bench/composer-arena/page.tsx | 41 +++- src/app/bench/composer-arena/types.ts | 28 ++- src/app/bench/page.tsx | 2 +- 18 files changed, 1051 insertions(+), 81 deletions(-) create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/README.md create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json create mode 100644 src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts create mode 100644 src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx diff --git a/package-lock.json b/package-lock.json index d088b57..4c67c1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", + "ajv": "^8.17.1", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", @@ -418,6 +419,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/js": { "version": "9.39.2", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", @@ -2139,16 +2164,16 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -3382,6 +3407,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -3497,6 +3546,23 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -4469,9 +4535,9 @@ "license": "MIT" }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, @@ -5503,6 +5569,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", diff --git a/package.json b/package.json index b0cdb8f..e74c361 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", + "ajv": "^8.17.1", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md new file mode 100644 index 0000000..10d4f9e --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md @@ -0,0 +1,17 @@ +# Composer Arena Phase 1 evidence pilot — 2026-07-19 + +This is a sanitized, dated deterministic-evidence packet. It is not a leaderboard, preference result, or production publication approval. + +The packet is bound to Composer Arena contract `composer-archive-to-output-arena-v0`, source head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7`, and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. + +It reports task outcomes and complete-output acceptance only: + +- mechanical control: pass and complete output; +- Grok 4.5: pass and complete output; +- Sol: `tool_denied` policy/task failure with no rendered candidate; +- Kimi K3: runtime/provider compatibility exclusion, not editing-quality loss; +- GLM 5.2: direct visual perception not applicable, not a failure. + +There are zero blind battles, zero human preference votes, zero ranks, and no winner. The required visual roster is incomplete, so the upstream Arena contract permits no Bradley-Terry observation. + +The public JSON contains no local/founder paths, media or source URLs, customer/GMA data, credentials, hidden-test material, private case annotations, or raw reasoning traces. diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json new file mode 100644 index 0000000..bd1cfb2 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json @@ -0,0 +1,8 @@ +{ + "algorithm": "sha256", + "files": { + "phase-1-results.json": "8525a64708f4da63ae0cf6dab3be5efa4786ad3466cbddf25ad6a462a0c0508a", + "phase-1-results.schema.json": "14d5ea76c9dbf2c9a6cc1fb8ece8659a3f069f3130f0bbcf4ffebc1a26e00f24", + "README.md": "08d34fe486b2956eec990c956204ad84a9ce362e9c0ddf03a89628017a34052f" + } +} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json new file mode 100644 index 0000000..6df2020 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json @@ -0,0 +1,161 @@ +{ + "schema_version": 1, + "evidence_id": "composer-arena-phase-1-pilot-2026-07-19", + "evidence_date": "2026-07-19", + "status": "phase-1-complete-publication-not-approved", + "data_class": "pilot_evidence", + "arena_binding": { + "arena_id": "composer-archive-to-output-arena-v0", + "source_head_sha": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", + "arena_contract_sha256": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", + "parent_contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" + }, + "publication": { + "surface": "evidence/pilot", + "publishable": false, + "state": "insufficient_evidence", + "leaderboard": false, + "winner": null, + "rank_count": 0, + "human_preference_claims": false, + "reasons": [ + "Only one complete model output exists for the audited cell.", + "The required visual roster is incomplete and therefore generates no preference battle.", + "The seven immutable holdout source families have not completed the frozen protocol." + ] + }, + "vote_provenance": { + "collection_mode": "no_eligible_votes", + "crowdsourced": false, + "planned_battle_groups": 0, + "blind_battles": 0, + "real_human_votes": 0, + "resolved_battle_groups": 0 + }, + "frozen_evidence": { + "dataset_repo_id": "HuggingFaceFV/finevideo", + "dataset_revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", + "selection_sha256": "39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4", + "release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + "case_pack_sha256": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203", + "content_evidence_sha256": "7f798fce08b0a2a2bf5d24e49f28f306bae87e0994b947d6c593ac2cdbba2128", + "task_brief_sha256": "45c58fca279847ce3029166a38814cfd4b13065bac93a6ddc71f24e480043569", + "tool_registry_sha256": "2f3a866c9ff9debc590f30361d6fabdbef0b808ded69beddd2054eac5795975c", + "runner": { + "version": "composer-system-eval-runner-v2", + "head_sha": "97d2606c98b2c38d3fbb0e338fbbebc2c6e8b22d", + "protocol_sha256": "5965228c9b0bbf0d8d46667b951d750f1dc1ffe9b5a3bb39b36fddbb39592d1b", + "prompt_sha256": "0278214adab1aea4c25681f64bcc66a8563910f51250cf6fbf04263faa563c37" + } + }, + "task_success": { + "separation_statement": "Deterministic task outcomes are evidence only and never enter a preference score.", + "results": [ + { + "baseline_id": "chronological-mechanical-v1", + "display_name": "Mechanical control", + "classification": "control", + "terminal_status": "passed", + "task_outcome": "pass", + "preference_eligible": false, + "quality_loss": false, + "run_id": "control-a4af853df6a1e23c6b10cfa9a84abcc4", + "latency_ms": null, + "failure_code": null, + "reason": "Zero-model chronological control completed the frozen task and output acceptance gates.", + "output": { + "complete": true, + "duration_ms": 45013, + "bytes": 22942811, + "video_codec": "H.264", + "audio_codec": "AAC", + "format": "1080p30", + "decode_passed": true, + "opening_black_ms": 0, + "av_start_drift_ms": 0, + "av_end_drift_ms": 13.333, + "output_sha256": "cb7a918b3c3f2b1eb44b944fa6d6a453cc71cd020e806274f976cf0a2901a2c4", + "render_acceptance_sha256": "279ee22dadbb53d861af7da139af4008e2b3bfd58da6fecb629b8b2f9bf0c6ad" + } + }, + { + "baseline_id": "grok-4.5-medium-v1", + "display_name": "Grok 4.5 medium", + "classification": "evaluated_model", + "terminal_status": "passed", + "task_outcome": "pass", + "preference_eligible": false, + "quality_loss": false, + "run_id": "rankable-4771baa518add8e98618bf579eac5976", + "latency_ms": 291034, + "failure_code": null, + "reason": "Complete output passed deterministic and render acceptance, but no second comparable roster output exists for a blind battle.", + "output": { + "complete": true, + "duration_ms": 58539, + "bytes": 27860030, + "video_codec": "H.264", + "audio_codec": "AAC", + "format": "1080p30", + "decode_passed": true, + "opening_black_ms": 0, + "av_start_drift_ms": 0, + "av_end_drift_ms": 5.334, + "output_sha256": "d8068df3d87c7e2cba6e609d098918e0cb705e9ac69cb6f2c45f23cd2000b671", + "render_acceptance_sha256": "607a2ce5da7b4a2cbdc81d3208a552d2164f7cd8775cc8d2c93ad53f25db878d" + } + }, + { + "baseline_id": "sol-medium-v1", + "display_name": "Sol medium", + "classification": "evaluated_model", + "terminal_status": "failed", + "task_outcome": "policy_task_failure", + "preference_eligible": false, + "quality_loss": false, + "run_id": "rankable-7053fc4cd2bbab5e6cd4ec5636ecd10e", + "latency_ms": 233774, + "failure_code": "tool_denied", + "reason": "The run attempted a direct timing-HTML mutation prohibited by the frozen runner-v2 tool policy; no rendered candidate or preference battle exists.", + "output": null + }, + { + "baseline_id": "kimi-k3-high-v1", + "display_name": "Kimi K3 high", + "classification": "evaluated_model", + "terminal_status": "excluded", + "task_outcome": "runtime_provider_compatibility", + "preference_eligible": false, + "quality_loss": false, + "run_id": "rankable-c00071c662b6192a996c161a10e0e62d", + "latency_ms": 2168, + "failure_code": null, + "reason": "Excluded at the runtime/provider compatibility boundary with zero tool calls; this is not an editing-quality loss.", + "output": null + }, + { + "baseline_id": "glm-5.2-high-v1", + "display_name": "GLM 5.2 high", + "classification": "not_applicable", + "terminal_status": "not_applicable", + "task_outcome": "direct_perception_unavailable", + "preference_eligible": false, + "quality_loss": false, + "run_id": null, + "latency_ms": null, + "failure_code": null, + "reason": "Direct visual perception is unavailable and GLM 5.2 remains outside the visual preference roster; this is not a failure.", + "output": null + } + ] + }, + "diagnostic_limitations": { + "hyperframes_version": "0.7.64", + "finding": "The renderer emitted non-blocking data-end/data-duration static-guard diagnostics even though browser, render, decode, duration, and A/V acceptance gates passed.", + "mechanical_control_diagnostic_count": 8, + "grok_4_5_diagnostic_count": 6, + "compatibility_treatment": "Preserve as a Composer/HyperFrames compatibility finding; do not reinterpret the passing outputs as failed.", + "auxiliary_description_step": "A snapshot command invoked an auxiliary Gemini description step unexpectedly. That output is excluded from evidence, scoring, and judgment." + }, + "next_gate": "all-seven-immutable-holdout-families" +} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json new file mode 100644 index 0000000..ae3e76f --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json @@ -0,0 +1,165 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json", + "title": "Composer Arena Phase 1 public pilot evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "evidence_id", + "evidence_date", + "status", + "data_class", + "arena_binding", + "publication", + "vote_provenance", + "frozen_evidence", + "task_success", + "diagnostic_limitations", + "next_gate" + ], + "properties": { + "schema_version": { "const": 1 }, + "evidence_id": { "const": "composer-arena-phase-1-pilot-2026-07-19" }, + "evidence_date": { "const": "2026-07-19" }, + "status": { "const": "phase-1-complete-publication-not-approved" }, + "data_class": { "const": "pilot_evidence" }, + "arena_binding": { + "type": "object", + "additionalProperties": false, + "required": ["arena_id", "source_head_sha", "arena_contract_sha256", "parent_contract_sha256"], + "properties": { + "arena_id": { "const": "composer-archive-to-output-arena-v0" }, + "source_head_sha": { "const": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7" }, + "arena_contract_sha256": { "const": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5" }, + "parent_contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" } + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["surface", "publishable", "state", "leaderboard", "winner", "rank_count", "human_preference_claims", "reasons"], + "properties": { + "surface": { "const": "evidence/pilot" }, + "publishable": { "const": false }, + "state": { "const": "insufficient_evidence" }, + "leaderboard": { "const": false }, + "winner": { "type": "null" }, + "rank_count": { "const": 0 }, + "human_preference_claims": { "const": false }, + "reasons": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } } + } + }, + "vote_provenance": { + "type": "object", + "additionalProperties": false, + "required": ["collection_mode", "crowdsourced", "planned_battle_groups", "blind_battles", "real_human_votes", "resolved_battle_groups"], + "properties": { + "collection_mode": { "const": "no_eligible_votes" }, + "crowdsourced": { "const": false }, + "planned_battle_groups": { "const": 0 }, + "blind_battles": { "const": 0 }, + "real_human_votes": { "const": 0 }, + "resolved_battle_groups": { "const": 0 } + } + }, + "frozen_evidence": { "$ref": "#/$defs/frozenEvidence" }, + "task_success": { + "type": "object", + "additionalProperties": false, + "required": ["separation_statement", "results"], + "properties": { + "separation_statement": { "type": "string", "minLength": 1 }, + "results": { + "type": "array", + "minItems": 5, + "maxItems": 5, + "items": { "$ref": "#/$defs/taskResult" } + } + } + }, + "diagnostic_limitations": { "$ref": "#/$defs/diagnosticLimitations" }, + "next_gate": { "const": "all-seven-immutable-holdout-families" } + }, + "$defs": { + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "sha1": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "frozenEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["dataset_repo_id", "dataset_revision", "selection_sha256", "release_sha256", "case_pack_sha256", "content_evidence_sha256", "task_brief_sha256", "tool_registry_sha256", "runner"], + "properties": { + "dataset_repo_id": { "const": "HuggingFaceFV/finevideo" }, + "dataset_revision": { "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" }, + "selection_sha256": { "$ref": "#/$defs/sha256" }, + "release_sha256": { "$ref": "#/$defs/sha256" }, + "case_pack_sha256": { "$ref": "#/$defs/sha256" }, + "content_evidence_sha256": { "$ref": "#/$defs/sha256" }, + "task_brief_sha256": { "$ref": "#/$defs/sha256" }, + "tool_registry_sha256": { "$ref": "#/$defs/sha256" }, + "runner": { + "type": "object", + "additionalProperties": false, + "required": ["version", "head_sha", "protocol_sha256", "prompt_sha256"], + "properties": { + "version": { "const": "composer-system-eval-runner-v2" }, + "head_sha": { "$ref": "#/$defs/sha1" }, + "protocol_sha256": { "$ref": "#/$defs/sha256" }, + "prompt_sha256": { "$ref": "#/$defs/sha256" } + } + } + } + }, + "taskResult": { + "type": "object", + "additionalProperties": false, + "required": ["baseline_id", "display_name", "classification", "terminal_status", "task_outcome", "preference_eligible", "quality_loss", "run_id", "latency_ms", "failure_code", "reason", "output"], + "properties": { + "baseline_id": { "enum": ["chronological-mechanical-v1", "grok-4.5-medium-v1", "sol-medium-v1", "kimi-k3-high-v1", "glm-5.2-high-v1"] }, + "display_name": { "type": "string", "minLength": 1 }, + "classification": { "enum": ["control", "evaluated_model", "not_applicable"] }, + "terminal_status": { "enum": ["passed", "failed", "excluded", "not_applicable"] }, + "task_outcome": { "enum": ["pass", "policy_task_failure", "runtime_provider_compatibility", "direct_perception_unavailable"] }, + "preference_eligible": { "const": false }, + "quality_loss": { "const": false }, + "run_id": { "type": ["string", "null"] }, + "latency_ms": { "type": ["integer", "null"], "minimum": 0 }, + "failure_code": { "type": ["string", "null"] }, + "reason": { "type": "string", "minLength": 1 }, + "output": { "oneOf": [{ "$ref": "#/$defs/output" }, { "type": "null" }] } + } + }, + "output": { + "type": "object", + "additionalProperties": false, + "required": ["complete", "duration_ms", "bytes", "video_codec", "audio_codec", "format", "decode_passed", "opening_black_ms", "av_start_drift_ms", "av_end_drift_ms", "output_sha256", "render_acceptance_sha256"], + "properties": { + "complete": { "const": true }, + "duration_ms": { "type": "integer", "minimum": 1 }, + "bytes": { "type": "integer", "minimum": 1 }, + "video_codec": { "type": "string", "minLength": 1 }, + "audio_codec": { "type": "string", "minLength": 1 }, + "format": { "type": "string", "minLength": 1 }, + "decode_passed": { "const": true }, + "opening_black_ms": { "type": "number", "minimum": 0 }, + "av_start_drift_ms": { "type": "number", "minimum": 0 }, + "av_end_drift_ms": { "type": "number", "minimum": 0 }, + "output_sha256": { "$ref": "#/$defs/sha256" }, + "render_acceptance_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "diagnosticLimitations": { + "type": "object", + "additionalProperties": false, + "required": ["hyperframes_version", "finding", "mechanical_control_diagnostic_count", "grok_4_5_diagnostic_count", "compatibility_treatment", "auxiliary_description_step"], + "properties": { + "hyperframes_version": { "const": "0.7.64" }, + "finding": { "type": "string", "minLength": 1 }, + "mechanical_control_diagnostic_count": { "const": 8 }, + "grok_4_5_diagnostic_count": { "const": 6 }, + "compatibility_treatment": { "type": "string", "minLength": 1 }, + "auxiliary_description_step": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/public/bench/composer-arena/v0/checksums.json b/public/bench/composer-arena/v0/checksums.json index 59b1181..a4204a4 100644 --- a/public/bench/composer-arena/v0/checksums.json +++ b/public/bench/composer-arena/v0/checksums.json @@ -1,11 +1,11 @@ { "algorithm": "sha256", "files": { - "release.schema.json": "a08a171fb29b3cbed0e173f549cf566597fb840168b34f06181e400d7fdd6ace", + "release.schema.json": "da3cb101dec52ac3cdeab1dc9e6f041668df150857d4781676e4b1b0399ae55f", "battle.schema.json": "b235183b07bf469172c79e8746275f2dd81dc30517667db5fb9a40f07c1f1572", - "release.preview.json": "bd4d755cc09154a5fed315299e8771908f3f956e2b63a8136b8544680dc01371", + "release.preview.json": "0d33d0aa14ed6295c11d23e81fe985af003941fecab12131e4f3fc3eee874915", "battles.preview.jsonl": "cb03fa3f39acc9c6488aa2beb45b08dc3cefe9fcaae236a66805873bd05d0d6c", - "methodology.md": "94ce70458152c1123a279181256c9a17d9321733c556317e54c65b2e85907917", - "handoff.md": "05524e739f8ce21e284ab83d4c54eba50b751b57f5b8ceaeeb0c05e95324305e" + "methodology.md": "b989ced1c20285f6257a9d3be1d11097fd29a11f6e4def030d03c194c2a29a55", + "handoff.md": "30f58c039880905d6fa6cad884c68417fd483af7d9ef0ef31b40cac2cf7beacb" } } diff --git a/public/bench/composer-arena/v0/handoff.md b/public/bench/composer-arena/v0/handoff.md index 6c7d5f1..ac65c46 100644 --- a/public/bench/composer-arena/v0/handoff.md +++ b/public/bench/composer-arena/v0/handoff.md @@ -2,6 +2,8 @@ The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. Real public artifacts can replace those files without changing React code when they preserve the schema and filenames below. +The canonical upstream contract binding is `composer-archive-to-output-arena-v0` at source head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7`, Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`, and parent system-eval contract SHA-256 `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624`. A future adapter must fail closed if any binding drifts. + ## Required files 1. `release.preview.json` — one release object validated against `release.schema.json`. @@ -11,12 +13,16 @@ The route reads `release.preview.json` and `battles.preview.jsonl` from this dir Before a rankable release, change `release_mode` to `governed_rankable_release` and `official_ranking_enabled` to `true` only after the already-green FineVideo release marker is joined by real runner records, a governed vote backend when votes are included, a frozen statistical policy, and command-room review. Synthetic or local/demo data must remain `synthetic_non_rankable_preview`. +The canonical upstream publication vocabulary is `ranked`, `tied`, and `insufficient_evidence`; public results have `scope: independent_track_results_only`. `controlled-semantic-text-evidence-v0` and `end-to-end-visual-orchestration-v0` remain separate, and a combined winner is forbidden. + +SEA Broadcast ASR remains a separate benchmark suite with its own rights, source, scoring, and publication contracts. Composer Arena artifacts must not change, absorb, or reinterpret SEA-ASR state. + ## Release rows Every track row must provide: -- `standing`: `ranked`, `tie`, or `insufficient_evidence`; -- `display_rank`: ordinal text, tie text such as `T1`, or `—`; +- `standing`: `ranked`, `tied`, or `insufficient_evidence`; +- `display_rank`: ordinal text only for a governed publishable release; synthetic/pilot evidence uses `—`; - model, provider, version, and evaluation date; - Arena score, two-element 95% confidence interval, and battle count; - deterministic task success and failure rate in `[0, 1]`; @@ -25,7 +31,7 @@ Every track row must provide: - estimated USD cost per run; - at least one per-task slice with task ID, label, success rate, and battle count. -`controlled-agent` and `end-to-end-system` must remain separate track objects. Exploratory capability cards must use `status: non_rankable` and cannot be folded into Arena score. +`controlled-agent` and `end-to-end-system` must remain separate track objects bound to the exact canonical track IDs above. Exploratory capability cards must use `status: non_rankable` and cannot be folded into Arena score. ## Battle JSONL contract @@ -46,5 +52,6 @@ The browser may swap left/right display order. Judgment choices are exactly `lef - No hidden test data, customer/GMA data, credentials, signed URLs, or raw reasoning traces enter public artifacts. - Every model route passes exact-provider/model/version preflight with no fallback substitution. - The statistical exporter emits ties and insufficient-evidence states rather than forcing ranks. +- Evidence/smoke partitions and pilot failures create deterministic evidence only; an incomplete required-roster stratum creates zero battles and no Bradley-Terry observation. - Official votes, if any, come only from a governed backend with a versioned inclusion policy. - Schema validation, checksum verification, lint, build, route contract checks, and HTTP/UI smoke are green. diff --git a/public/bench/composer-arena/v0/methodology.md b/public/bench/composer-arena/v0/methodology.md index 121c41f..a2db4c0 100644 --- a/public/bench/composer-arena/v0/methodology.md +++ b/public/bench/composer-arena/v0/methodology.md @@ -1,18 +1,18 @@ # Composer Archive-to-Output Arena v0 methodology -Status: **non-rankable preview**. The checked-in metrics and battle outputs are synthetic interface fixtures, not measured model results. +Status: **non-rankable preview**. The checked-in metrics and battle outputs are synthetic interface fixtures, not measured model results. This public adapter is bound to `PixelML/agentic_video_intelligence` head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. ## Tracks -- `controlled-agent` compares evidence-grounded archive understanding, temporal reasoning, and executable edit planning over the same frozen evidence and brief. -- `end-to-end-system` compares complete Composer executions over the same released cell and brief. Timeline mutation must use the frozen Composer tool contract and pass structural gates. +- `controlled-semantic-text-evidence-v0` / `controlled-agent` requires all four systems over one genuine, hash-identical transcript/OCR/semantic-event artifact. That artifact is unavailable, so the real v0 track is blocked and the visible rows are synthetic interface examples only. +- `end-to-end-visual-orchestration-v0` / `end-to-end-system` requires Sol, Grok 4.5, and Kimi K3 over the same visual case pack. GLM 5.2 direct perception is explicitly not applicable and remains outside that roster. - The two tracks are reported separately and never collapsed into one leaderboard. ## Standing and uncertainty -Arena score is a presentation field for the future governed pairwise estimator. A real release must document the estimator, seed, bootstrap procedure, tie threshold, and vote inclusion policy. The public surface shows the frozen 95% confidence interval and battle count beside every score. +The canonical Arena score is Bradley-Terry estimated win probability against Sol, normalized so Sol is exactly 50.0. Uncertainty uses 2,000 deterministic source-family cluster-bootstrap replicates at 95% confidence. The public preview demonstrates that presentation with synthetic numbers only. -An ordinal standing is forbidden when the minimum battle count is not met or required evidence is incomplete. A tie is displayed when the governed comparison cannot establish separation at 95% confidence. Synthetic preview rows remain non-rankable regardless of their displayed fixture standing. +An ordinal standing is forbidden when the frozen battle, seven-source-family, human-judge, complete-roster, connectivity, bootstrap, contamination, or position-bias gates are not met. `tied` means a pairwise 95% interval contains zero; `insufficient_evidence` means one or more gates failed. Synthetic preview rows remain `publishable=false`, `rank_count=0`, and non-rankable regardless of their illustrative state. Deterministic task success is computed independently from preference battles. Failure rate retains explicit terminal failures in the denominator. Latency, token totals, and estimated cost use frozen run records; they never disappear because a run failed. @@ -32,6 +32,8 @@ Public-eval and private-test source families must remain disjoint. Hidden-test f ## Frozen evidence +- Arena source head: `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` +- Arena contract SHA-256: `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5` - System-eval contract SHA-256: `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624` - FineVideo selection release SHA-256: `d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926` - FineVideo selection-manifest SHA-256: `39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4` @@ -47,7 +49,8 @@ Public exports may contain aggregate metrics, public-safe artifact hashes, requi ## Known limitations - All preview scores, intervals, costs, and outputs are synthetic. -- The FineVideo source release is green, but the rankable runner/runtime is still being repaired and has produced no public model records. +- Phase 1 has one complete model output plus a mechanical control, but no complete seven-family required roster and therefore no blind battle or preference score. +- The dated Phase 1 pilot evidence is deterministic task-success evidence with zero blind battles, zero ranks, and zero human preference claims. - The public surface does not implement an official vote backend. - The frozen system-eval contract currently uses one repetition per real cell; reliability claims need a reviewed repeated-run extension. - Kimi K3 uses Composer's expressible `high` effort because the runtime cannot express the provider's published `max` effort. diff --git a/public/bench/composer-arena/v0/release.preview.json b/public/bench/composer-arena/v0/release.preview.json index 3f274f3..6a640ed 100644 --- a/public/bench/composer-arena/v0/release.preview.json +++ b/public/bench/composer-arena/v0/release.preview.json @@ -4,16 +4,34 @@ "release_id": "composer-arena-v0-synthetic-preview-2026-07-19", "generated_at": "2026-07-19T12:00:00Z", "release_mode": "synthetic_non_rankable_preview", + "data_class": "synthetic", "official_ranking_enabled": false, "official_vote_backend": "not_configured", "display_label": "non-rankable preview", + "arena_binding": { + "arena_id": "composer-archive-to-output-arena-v0", + "source_head_sha": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", + "arena_contract_sha256": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", + "parent_contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" + }, + "publication": { + "publishable": false, + "state": "insufficient_evidence", + "scope": "independent_track_results_only", + "winner": null, + "rank_count": 0 + }, + "vote_provenance": { + "collection_mode": "synthetic_fixture", + "crowdsourced": false, + "real_human_votes": 0 + }, "dataset": { "repo_id": "HuggingFaceFV/finevideo", "revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", "license_claim": "CC-BY", "license_scope": "The 24 command-room-audited public-eval cells carry CC-BY publisher metadata in the frozen selection manifest. Each real public example must retain its source-specific attribution text.", "attribution": "FineVideo, HuggingFaceFV/finevideo at immutable revision 84c74091e1c6ee7a5dffabfafb5c9033e4718883. Source-specific creator attribution is retained per released cell.", - "contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", "finevideo_release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", "selection_manifest_sha256": "39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4", "public_eval_source_family_count": 24, @@ -25,8 +43,11 @@ } }, "evaluation_policy": { + "ranking_method": "Bradley-Terry logistic maximum likelihood", "confidence_level": 0.95, - "minimum_battles_for_standing": 20, + "bootstrap_replicates": 2000, + "minimum_resolved_battles_per_pair_overall": 4, + "position_bias_decisive_presentations": 20, "tie_rule": "Show a tie when the frozen comparison cannot establish a separated standing at 95% confidence.", "insufficient_evidence_rule": "Do not assign an ordinal standing below the minimum battle count or when required evidence is incomplete.", "local_vote_policy": "Replay judgments are browser-local demonstrations. They are not transmitted, persisted, or included in official standings.", @@ -36,18 +57,19 @@ "tracks": [ { "id": "controlled-agent", + "canonical_track_id": "controlled-semantic-text-evidence-v0", "title": "Controlled agent", - "description": "Same frozen archive evidence and task brief; scores evidence-grounded planning and temporal reasoning without running the full production system.", + "description": "Synthetic interface examples only. The real controlled-semantic track is blocked until one genuine, hash-identical transcript/OCR/semantic-event artifact is frozen for all four systems.", "rows": [ { - "standing": "tie", - "display_rank": "T1", + "standing": "tied", + "display_rank": "—", "model": "GPT-5.6 Sol", "provider": "AgenticFlow", "version": "sol-medium-v1", "evaluation_date": "2026-07-19", - "arena_score": 1074, - "arena_score_ci95": [1051, 1096], + "arena_score": 50, + "arena_score_ci95": [47.7, 52.3], "battle_count": 48, "deterministic_task_success": 0.833, "failure_rate": 0.063, @@ -61,14 +83,14 @@ ] }, { - "standing": "tie", - "display_rank": "T1", + "standing": "tied", + "display_rank": "—", "model": "Grok 4.5", "provider": "AgenticFlow", "version": "grok-4.5-medium-v1", "evaluation_date": "2026-07-19", - "arena_score": 1068, - "arena_score_ci95": [1046, 1090], + "arena_score": 52.4, + "arena_score_ci95": [49.7, 55.1], "battle_count": 48, "deterministic_task_success": 0.813, "failure_rate": 0.083, @@ -88,8 +110,8 @@ "provider": "AgenticFlow", "version": "kimi-k3-high-v1", "evaluation_date": "2026-07-19", - "arena_score": 1059, - "arena_score_ci95": [982, 1131], + "arena_score": 49.1, + "arena_score_ci95": [41.2, 57], "battle_count": 8, "deterministic_task_success": 0.75, "failure_rate": 0.125, @@ -106,18 +128,19 @@ }, { "id": "end-to-end-system", + "canonical_track_id": "end-to-end-visual-orchestration-v0", "title": "End-to-end system", - "description": "Same released cells and brief, executed through Composer tooling and structural gates; never blended with the controlled-agent table.", + "description": "Synthetic interface examples for the separate visual-orchestration track. Real results require complete Sol, Grok 4.5, and Kimi K3 outputs on every frozen holdout stratum.", "rows": [ { - "standing": "tie", - "display_rank": "T1", + "standing": "tied", + "display_rank": "—", "model": "GPT-5.6 Sol + Composer", "provider": "AgenticFlow / Pixel ML", "version": "sol-medium-v1 / composer-v0", "evaluation_date": "2026-07-19", - "arena_score": 1042, - "arena_score_ci95": [1014, 1071], + "arena_score": 50, + "arena_score_ci95": [46.9, 53.1], "battle_count": 36, "deterministic_task_success": 0.778, "failure_rate": 0.111, @@ -131,14 +154,14 @@ ] }, { - "standing": "tie", - "display_rank": "T1", + "standing": "tied", + "display_rank": "—", "model": "Grok 4.5 + Composer", "provider": "AgenticFlow / Pixel ML", "version": "grok-4.5-medium-v1 / composer-v0", "evaluation_date": "2026-07-19", - "arena_score": 1036, - "arena_score_ci95": [1008, 1065], + "arena_score": 51.6, + "arena_score_ci95": [48.4, 54.8], "battle_count": 36, "deterministic_task_success": 0.75, "failure_rate": 0.139, @@ -154,12 +177,12 @@ { "standing": "insufficient_evidence", "display_rank": "—", - "model": "GLM 5.2 + Composer", + "model": "Kimi K3 + Composer", "provider": "AgenticFlow / Pixel ML", - "version": "glm-5.2-high-v1 / composer-v0", + "version": "kimi-k3-high-v1 / composer-v0", "evaluation_date": "2026-07-19", - "arena_score": 1028, - "arena_score_ci95": [946, 1112], + "arena_score": 48.8, + "arena_score_ci95": [39.6, 58.1], "battle_count": 6, "deterministic_task_success": 0.667, "failure_rate": 0.167, @@ -203,7 +226,7 @@ ], "known_limitations": [ "All scores, confidence intervals, battle counts, costs, and outputs in this preview are synthetic interface fixtures and must not be interpreted as measured model performance.", - "The FineVideo selection and FINEVIDEO_RELEASED marker pass the 24-cell release gate, but the rankable runner/runtime has not produced real model records; every displayed row remains synthetic and non-rankable.", + "Phase 1 produced one complete model output plus a mechanical control, but no complete seven-family required roster or blind human comparison; every preview-table row remains synthetic and non-rankable.", "One repetition per real cell is currently frozen in the system-eval contract; reliability claims require a reviewed repeated-run extension.", "Kimi K3 is evaluated at Composer's expressible high effort because the runtime cannot express the provider's published max effort.", "Local replay judgments are browser-only and have no governed identity, anti-abuse, sampling, or persistence backend.", diff --git a/public/bench/composer-arena/v0/release.schema.json b/public/bench/composer-arena/v0/release.schema.json index 300f3b2..ec35d7c 100644 --- a/public/bench/composer-arena/v0/release.schema.json +++ b/public/bench/composer-arena/v0/release.schema.json @@ -10,9 +10,13 @@ "release_id", "generated_at", "release_mode", + "data_class", "official_ranking_enabled", "official_vote_backend", "display_label", + "arena_binding", + "publication", + "vote_provenance", "dataset", "evaluation_policy", "tracks", @@ -27,9 +31,13 @@ "release_mode": { "enum": ["synthetic_non_rankable_preview", "governed_rankable_release"] }, + "data_class": { "enum": ["rankable", "exploratory", "synthetic"] }, "official_ranking_enabled": { "type": "boolean" }, "official_vote_backend": { "enum": ["not_configured", "governed_v1"] }, "display_label": { "type": "string", "minLength": 1 }, + "arena_binding": { "$ref": "#/$defs/arenaBinding" }, + "publication": { "$ref": "#/$defs/publication" }, + "vote_provenance": { "$ref": "#/$defs/voteProvenance" }, "dataset": { "$ref": "#/$defs/dataset" }, "evaluation_policy": { "$ref": "#/$defs/evaluationPolicy" }, "tracks": { @@ -63,7 +71,6 @@ "license_claim", "license_scope", "attribution", - "contract_sha256", "finevideo_release_sha256", "selection_manifest_sha256", "public_eval_source_family_count", @@ -77,7 +84,6 @@ "license_claim": { "const": "CC-BY" }, "license_scope": { "type": "string", "minLength": 1 }, "attribution": { "type": "string", "minLength": 1 }, - "contract_sha256": { "$ref": "#/$defs/sha256" }, "finevideo_release_sha256": { "$ref": "#/$defs/sha256" }, "selection_manifest_sha256": { "$ref": "#/$defs/sha256" }, "public_eval_source_family_count": { "type": "integer", "minimum": 1 }, @@ -103,8 +109,11 @@ "type": "object", "additionalProperties": false, "required": [ + "ranking_method", "confidence_level", - "minimum_battles_for_standing", + "bootstrap_replicates", + "minimum_resolved_battles_per_pair_overall", + "position_bias_decisive_presentations", "tie_rule", "insufficient_evidence_rule", "local_vote_policy", @@ -112,8 +121,11 @@ "privacy_limit" ], "properties": { + "ranking_method": { "const": "Bradley-Terry logistic maximum likelihood" }, "confidence_level": { "const": 0.95 }, - "minimum_battles_for_standing": { "type": "integer", "minimum": 1 }, + "bootstrap_replicates": { "const": 2000 }, + "minimum_resolved_battles_per_pair_overall": { "const": 4 }, + "position_bias_decisive_presentations": { "const": 20 }, "tie_rule": { "type": "string", "minLength": 1 }, "insufficient_evidence_rule": { "type": "string", "minLength": 1 }, "local_vote_policy": { "type": "string", "minLength": 1 }, @@ -124,9 +136,10 @@ "track": { "type": "object", "additionalProperties": false, - "required": ["id", "title", "description", "rows"], + "required": ["id", "canonical_track_id", "title", "description", "rows"], "properties": { "id": { "enum": ["controlled-agent", "end-to-end-system"] }, + "canonical_track_id": { "enum": ["controlled-semantic-text-evidence-v0", "end-to-end-visual-orchestration-v0"] }, "title": { "type": "string", "minLength": 1 }, "description": { "type": "string", "minLength": 1 }, "rows": { @@ -157,18 +170,18 @@ "per_task_slices" ], "properties": { - "standing": { "enum": ["ranked", "tie", "insufficient_evidence"] }, + "standing": { "enum": ["ranked", "tied", "insufficient_evidence"] }, "display_rank": { "type": "string", "minLength": 1 }, "model": { "type": "string", "minLength": 1 }, "provider": { "type": "string", "minLength": 1 }, "version": { "type": "string", "minLength": 1 }, "evaluation_date": { "type": "string", "format": "date" }, - "arena_score": { "type": "number" }, + "arena_score": { "type": "number", "minimum": 0, "maximum": 100 }, "arena_score_ci95": { "type": "array", "minItems": 2, "maxItems": 2, - "prefixItems": [{ "type": "number" }, { "type": "number" }] + "prefixItems": [{ "type": "number", "minimum": 0, "maximum": 100 }, { "type": "number", "minimum": 0, "maximum": 100 }] }, "battle_count": { "type": "integer", "minimum": 0 }, "deterministic_task_success": { "type": "number", "minimum": 0, "maximum": 1 }, @@ -205,6 +218,39 @@ "battle_count": { "type": "integer", "minimum": 0 } } }, + "arenaBinding": { + "type": "object", + "additionalProperties": false, + "required": ["arena_id", "source_head_sha", "arena_contract_sha256", "parent_contract_sha256"], + "properties": { + "arena_id": { "const": "composer-archive-to-output-arena-v0" }, + "source_head_sha": { "const": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7" }, + "arena_contract_sha256": { "const": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5" }, + "parent_contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" } + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["publishable", "state", "scope", "winner", "rank_count"], + "properties": { + "publishable": { "type": "boolean" }, + "state": { "enum": ["ranked", "tied", "insufficient_evidence"] }, + "scope": { "const": "independent_track_results_only" }, + "winner": { "type": ["string", "null"] }, + "rank_count": { "type": "integer", "minimum": 0 } + } + }, + "voteProvenance": { + "type": "object", + "additionalProperties": false, + "required": ["collection_mode", "crowdsourced", "real_human_votes"], + "properties": { + "collection_mode": { "enum": ["bounded_human_review", "exploratory_model_review", "synthetic_fixture", "no_eligible_votes"] }, + "crowdsourced": { "const": false }, + "real_human_votes": { "type": "integer", "minimum": 0 } + } + }, "exploratoryCapability": { "type": "object", "additionalProperties": false, diff --git a/scripts/check-composer-arena.mjs b/scripts/check-composer-arena.mjs index 929c15c..4ceecb0 100644 --- a/scripts/check-composer-arena.mjs +++ b/scripts/check-composer-arena.mjs @@ -1,10 +1,12 @@ import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { resolve } from "node:path"; +import Ajv2020 from "ajv/dist/2020.js"; const root = resolve(import.meta.dirname, ".."); const routeRoot = resolve(root, "src/app/bench/composer-arena"); const artifactRoot = resolve(root, "public/bench/composer-arena/v0"); +const evidenceArtifactRoot = resolve(root, "public/bench/composer-arena/evidence/pilot/2026-07-19"); const release = JSON.parse(readFileSync(resolve(artifactRoot, "release.preview.json"), "utf8")); const schema = JSON.parse(readFileSync(resolve(artifactRoot, "release.schema.json"), "utf8")); const checksums = JSON.parse(readFileSync(resolve(artifactRoot, "checksums.json"), "utf8")); @@ -18,13 +20,24 @@ const client = readFileSync(resolve(routeRoot, "arena-client.tsx"), "utf8"); const data = readFileSync(resolve(routeRoot, "data.ts"), "utf8"); const methodology = readFileSync(resolve(artifactRoot, "methodology.md"), "utf8"); const handoff = readFileSync(resolve(artifactRoot, "handoff.md"), "utf8"); +const evidence = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "phase-1-results.json"), "utf8")); +const evidenceSchema = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "phase-1-results.schema.json"), "utf8")); +const evidenceChecksums = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "checksums.json"), "utf8")); +const evidencePage = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/page.tsx"), "utf8"); +const evidenceData = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/data.ts"), "utf8"); function requireCondition(condition, message) { if (!condition) throw new Error(message); } -function sha256(filename) { - return createHash("sha256").update(readFileSync(resolve(artifactRoot, filename))).digest("hex"); +function validateSchema(schemaDocument, value, label) { + const ajv = new Ajv2020({ allErrors: true, strict: false, validateFormats: false }); + const validate = ajv.compile(schemaDocument); + requireCondition(validate(value), `${label} schema validation failed: ${ajv.errorsText(validate.errors)}`); +} + +function sha256(base, filename) { + return createHash("sha256").update(readFileSync(resolve(base, filename))).digest("hex"); } const requiredArtifacts = [ @@ -45,19 +58,34 @@ requireCondition(page.includes('const ARTIFACT_ROOT = "/bench/composer-arena/v0" for (const [filename, expected] of Object.entries(checksums.files)) { requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid checksum for ${filename}`); - requireCondition(sha256(filename) === expected, `checksum mismatch for ${filename}`); + requireCondition(sha256(artifactRoot, filename) === expected, `checksum mismatch for ${filename}`); +} + +for (const [filename, expected] of Object.entries(evidenceChecksums.files)) { + requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid pilot checksum for ${filename}`); + requireCondition(sha256(evidenceArtifactRoot, filename) === expected, `pilot checksum mismatch for ${filename}`); } requireCondition(schema.$id === "https://agentic.video/bench/composer-arena/v0/release.schema.json", "schema ID drifted"); requireCondition(schema.additionalProperties === false, "release schema must fail closed on unknown top-level fields"); +validateSchema(schema, release, "preview release"); +const battleSchema = JSON.parse(readFileSync(resolve(artifactRoot, "battle.schema.json"), "utf8")); +for (const battle of battles) validateSchema(battleSchema, battle, `battle ${battle.battle_id}`); requireCondition(release.release_mode === "synthetic_non_rankable_preview", "fixtures must remain a synthetic preview"); +requireCondition(release.data_class === "synthetic", "preview data class must remain synthetic"); requireCondition(release.official_ranking_enabled === false, "official ranking must remain disabled for fixtures"); requireCondition(release.official_vote_backend === "not_configured", "preview must not imply a governed vote backend"); requireCondition(release.display_label === "non-rankable preview", "preview label drifted"); requireCondition(release.dataset.repo_id === "HuggingFaceFV/finevideo", "FineVideo repo drifted"); requireCondition(release.dataset.revision === "84c74091e1c6ee7a5dffabfafb5c9033e4718883", "FineVideo revision drifted"); requireCondition(release.dataset.license_claim === "CC-BY", "FineVideo licence claim drifted"); -requireCondition(release.dataset.contract_sha256 === "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", "contract hash drifted"); +requireCondition(release.arena_binding.source_head_sha === "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", "Arena source head drifted"); +requireCondition(release.arena_binding.arena_contract_sha256 === "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", "Arena contract hash drifted"); +requireCondition(release.arena_binding.parent_contract_sha256 === "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", "parent contract hash drifted"); +requireCondition(release.publication.publishable === false, "synthetic preview must not be publishable"); +requireCondition(release.publication.state === "insufficient_evidence", "synthetic preview publication state must fail closed"); +requireCondition(release.publication.winner === null && release.publication.rank_count === 0, "synthetic preview must have no winner or rank"); +requireCondition(release.vote_provenance.collection_mode === "synthetic_fixture" && release.vote_provenance.real_human_votes === 0, "synthetic preview vote provenance drifted"); requireCondition(release.dataset.finevideo_release_sha256 === "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", "FineVideo release hash drifted"); requireCondition(release.dataset.split_policy.source_family_disjoint === true, "source families must remain disjoint"); requireCondition(release.dataset.split_policy.hidden_test_published === false, "hidden test must remain unpublished"); @@ -67,10 +95,15 @@ requireCondition( JSON.stringify(trackIds) === JSON.stringify(["controlled-agent", "end-to-end-system"]), "release must contain exactly the two separate Arena tracks", ); +const canonicalTrackIds = release.tracks.map((track) => track.canonical_track_id).sort(); +requireCondition( + JSON.stringify(canonicalTrackIds) === JSON.stringify(["controlled-semantic-text-evidence-v0", "end-to-end-visual-orchestration-v0"]), + "canonical Arena track IDs drifted", +); for (const track of release.tracks) { requireCondition(track.rows.length > 0, `${track.id} has no rows`); - requireCondition(track.rows.some((row) => row.standing === "tie"), `${track.id} must show a visible tie`); + requireCondition(track.rows.some((row) => row.standing === "tied"), `${track.id} must show a visible tied example`); requireCondition( track.rows.some((row) => row.standing === "insufficient_evidence"), `${track.id} must show insufficient evidence`, @@ -78,7 +111,7 @@ for (const track of release.tracks) { for (const row of track.rows) { requireCondition(row.model && row.provider && row.version && row.evaluation_date, `${track.id} row identity is incomplete`); - requireCondition(Number.isFinite(row.arena_score), `${track.id} row is missing Arena score`); + requireCondition(Number.isFinite(row.arena_score) && row.arena_score >= 0 && row.arena_score <= 100, `${track.id} row has an invalid preview score`); requireCondition(Array.isArray(row.arena_score_ci95) && row.arena_score_ci95.length === 2, `${track.id} row CI is invalid`); requireCondition(Number.isInteger(row.battle_count) && row.battle_count >= 0, `${track.id} battle count is invalid`); requireCondition(row.deterministic_task_success >= 0 && row.deterministic_task_success <= 1, `${track.id} task success is invalid`); @@ -114,6 +147,45 @@ requireCondition(methodology.includes("runner/runtime"), "methodology must discl requireCondition(handoff.includes("without changing React code"), "handoff must preserve schema-driven replacement"); requireCondition(handoff.includes("battle.schema.json"), "handoff must bind JSONL rows to the battle schema"); requireCondition(handoff.includes("governed vote backend"), "handoff is missing official vote governance"); +requireCondition(handoff.includes("fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5"), "handoff is not bound to the Arena contract"); +requireCondition(handoff.includes("controlled-semantic-text-evidence-v0") && handoff.includes("end-to-end-visual-orchestration-v0"), "handoff is missing canonical track separation"); +requireCondition(handoff.includes("SEA Broadcast ASR remains a separate benchmark suite"), "handoff must preserve the SEA-ASR boundary"); requireCondition(!page.includes("customer data") || page.includes("never"), "page must not normalize customer data use"); +requireCondition(evidenceSchema.additionalProperties === false, "pilot evidence schema must fail closed"); +validateSchema(evidenceSchema, evidence, "Phase 1 pilot evidence"); +requireCondition(evidence.arena_binding.source_head_sha === "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", "pilot source head drifted"); +requireCondition(evidence.arena_binding.arena_contract_sha256 === "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", "pilot Arena hash drifted"); +requireCondition(evidence.publication.publishable === false, "pilot evidence must not be publishable"); +requireCondition(evidence.publication.state === "insufficient_evidence", "pilot evidence must remain insufficient"); +requireCondition(evidence.publication.leaderboard === false, "pilot evidence must not be a leaderboard"); +requireCondition(evidence.publication.winner === null, "pilot evidence must not imply a winner"); +requireCondition(evidence.publication.rank_count === 0, "pilot evidence must publish zero ranks"); +requireCondition(evidence.publication.human_preference_claims === false, "pilot evidence must publish zero human preference claims"); +requireCondition( + evidence.vote_provenance.blind_battles === 0 && + evidence.vote_provenance.real_human_votes === 0 && + evidence.vote_provenance.resolved_battle_groups === 0, + "pilot evidence must publish zero battles and votes", +); + +const pilotResults = Object.fromEntries(evidence.task_success.results.map((result) => [result.baseline_id, result])); +requireCondition(pilotResults["chronological-mechanical-v1"].terminal_status === "passed" && pilotResults["chronological-mechanical-v1"].output?.complete === true, "mechanical control evidence drifted"); +requireCondition(pilotResults["grok-4.5-medium-v1"].terminal_status === "passed" && pilotResults["grok-4.5-medium-v1"].output?.complete === true, "Grok Phase 1 evidence drifted"); +requireCondition(pilotResults["sol-medium-v1"].failure_code === "tool_denied" && pilotResults["sol-medium-v1"].task_outcome === "policy_task_failure", "Sol policy/task failure drifted"); +requireCondition(pilotResults["kimi-k3-high-v1"].terminal_status === "excluded" && pilotResults["kimi-k3-high-v1"].quality_loss === false, "Kimi compatibility exclusion drifted"); +requireCondition(pilotResults["glm-5.2-high-v1"].terminal_status === "not_applicable" && pilotResults["glm-5.2-high-v1"].quality_loss === false, "GLM direct-vision N/A drifted"); +requireCondition(evidence.diagnostic_limitations.hyperframes_version === "0.7.64", "HyperFrames diagnostic version drifted"); +requireCondition(evidence.diagnostic_limitations.mechanical_control_diagnostic_count === 8 && evidence.diagnostic_limitations.grok_4_5_diagnostic_count === 6, "HyperFrames diagnostic counts drifted"); + +const serializedEvidence = JSON.stringify(evidence); +for (const forbidden of ["/Users/", "/tmp/", "http://", "https://", "s3://", "signed_url", "credential", "password", "secret", "hidden_test"]) { + requireCondition(!serializedEvidence.toLowerCase().includes(forbidden.toLowerCase()), `pilot evidence contains forbidden public material: ${forbidden}`); +} +requireCondition(evidencePage.includes("not a leaderboard"), "pilot route must visibly reject leaderboard framing"); +requireCondition(evidencePage.includes("Winner: none"), "pilot route must visibly reject a winner claim"); +requireCondition(evidencePage.includes("Blind battles") && evidencePage.includes("Published ranks") && evidencePage.includes("Human preference claims"), "pilot route must show all zero claim counts"); +requireCondition(evidencePage.includes("HyperFrames compatibility finding"), "pilot route must disclose the HyperFrames limitation"); +requireCondition(evidenceData.includes("phase-1-results.json"), "pilot route must be artifact-driven"); + console.log("Composer Arena route contract check passed"); diff --git a/src/app/bench/composer-arena/arena-client.tsx b/src/app/bench/composer-arena/arena-client.tsx index 608986e..bf85a96 100644 --- a/src/app/bench/composer-arena/arena-client.tsx +++ b/src/app/bench/composer-arena/arena-client.tsx @@ -18,7 +18,7 @@ const percent = new Intl.NumberFormat("en-US", { const integer = new Intl.NumberFormat("en-US"); -function StandingBadge({ standing }: { standing: "ranked" | "tie" | "insufficient_evidence" }) { +function StandingBadge({ standing }: { standing: "ranked" | "tied" | "insufficient_evidence" }) { if (standing === "insufficient_evidence") { return ( @@ -27,10 +27,10 @@ function StandingBadge({ standing }: { standing: "ranked" | "tie" | "insufficien ); } - if (standing === "tie") { + if (standing === "tied") { return ( - tie + tied example ); } @@ -52,6 +52,7 @@ function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: Aren

{track.title}

{track.description}

+ {track.canonical_track_id}
@@ -62,7 +63,7 @@ function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: Aren {[ "Standing", "Model / route", - "Arena score (95% CI)", + "Preview score vs Sol (95% CI)", "Battles", "Task success", "Failure rate", @@ -318,7 +319,7 @@ export default function ArenaClient({ release, battles }: { release: ArenaReleas
- Synthetic fixture values only. Ties are explicit; rows below {release.evaluation_policy.minimum_battles_for_standing} governed battles show insufficient evidence instead of an ordinal rank. + Synthetic preview examples only. They use PR #104's 0–100 score-vs-Sol presentation, but publish no rank; real evidence must satisfy at least {release.evaluation_policy.minimum_resolved_battles_per_pair_overall} resolved battles per pair plus the full frozen gates.
diff --git a/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts new file mode 100644 index 0000000..bff6320 --- /dev/null +++ b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts @@ -0,0 +1,116 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +export type PhaseOneOutput = { + complete: true; + duration_ms: number; + bytes: number; + video_codec: string; + audio_codec: string; + format: string; + decode_passed: true; + opening_black_ms: number; + av_start_drift_ms: number; + av_end_drift_ms: number; + output_sha256: string; + render_acceptance_sha256: string; +}; + +export type PhaseOneResult = { + baseline_id: string; + display_name: string; + classification: "control" | "evaluated_model" | "not_applicable"; + terminal_status: "passed" | "failed" | "excluded" | "not_applicable"; + task_outcome: "pass" | "policy_task_failure" | "runtime_provider_compatibility" | "direct_perception_unavailable"; + preference_eligible: false; + quality_loss: false; + run_id: string | null; + latency_ms: number | null; + failure_code: string | null; + reason: string; + output: PhaseOneOutput | null; +}; + +export type PhaseOneEvidence = { + evidence_id: string; + evidence_date: string; + status: string; + arena_binding: { + arena_id: string; + source_head_sha: string; + arena_contract_sha256: string; + parent_contract_sha256: string; + }; + publication: { + publishable: false; + state: "insufficient_evidence"; + leaderboard: false; + winner: null; + rank_count: 0; + human_preference_claims: false; + reasons: string[]; + }; + vote_provenance: { + planned_battle_groups: 0; + blind_battles: 0; + real_human_votes: 0; + resolved_battle_groups: 0; + }; + frozen_evidence: { + dataset_repo_id: string; + dataset_revision: string; + selection_sha256: string; + release_sha256: string; + case_pack_sha256: string; + content_evidence_sha256: string; + task_brief_sha256: string; + tool_registry_sha256: string; + runner: { + version: string; + head_sha: string; + protocol_sha256: string; + prompt_sha256: string; + }; + }; + task_success: { + separation_statement: string; + results: PhaseOneResult[]; + }; + diagnostic_limitations: { + hyperframes_version: string; + finding: string; + mechanical_control_diagnostic_count: number; + grok_4_5_diagnostic_count: number; + compatibility_treatment: string; + auxiliary_description_step: string; + }; + next_gate: string; +}; + +type Checksums = { + algorithm: "sha256"; + files: Record; +}; + +const artifactRoot = path.join( + process.cwd(), + "public", + "bench", + "composer-arena", + "evidence", + "pilot", + "2026-07-19", +); + +export async function loadPhaseOneEvidence() { + const [evidence, checksums] = await Promise.all([ + readFile(path.join(artifactRoot, "phase-1-results.json"), "utf8").then( + (raw) => JSON.parse(raw) as PhaseOneEvidence, + ), + readFile(path.join(artifactRoot, "checksums.json"), "utf8").then( + (raw) => JSON.parse(raw) as Checksums, + ), + ]); + + return { evidence, checksums }; +} diff --git a/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx new file mode 100644 index 0000000..25c2efa --- /dev/null +++ b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx @@ -0,0 +1,227 @@ +import type { Metadata } from "next"; +import Link from "next/link"; + +import { loadPhaseOneEvidence, type PhaseOneResult } from "./data"; + +const ARTIFACT_ROOT = "/bench/composer-arena/evidence/pilot/2026-07-19"; + +export const metadata: Metadata = { + title: "Composer Arena Phase 1 evidence pilot — July 19, 2026", + description: + "Sanitized deterministic Composer Phase 1 evidence: zero blind battles, zero ranks, and no human preference claim.", + alternates: { canonical: "/bench/composer-arena/evidence/pilot/2026-07-19" }, +}; + +function HashValue({ children }: { children: string }) { + return {children}; +} + +function statusClasses(result: PhaseOneResult) { + if (result.terminal_status === "passed") return "border-emerald-300 bg-emerald-50 text-emerald-950"; + if (result.terminal_status === "failed") return "border-red-300 bg-red-50 text-red-950"; + return "border-amber-300 bg-amber-50 text-amber-950"; +} + +export default async function PhaseOneEvidencePage() { + const { evidence, checksums } = await loadPhaseOneEvidence(); + const completedOutputs = evidence.task_success.results.filter((result) => result.output); + + return ( +
+
+
+
+ av + + Composer Arena + +
+ + Evidence / pilot · 2026-07-19 + +
+
+ +
+
+ + not a leaderboard + +

+ Composer Archive-to-Output Arena v0 +

+

+ Phase 1 evidence pilot +

+

+ A sanitized, dated record of deterministic task outcomes and complete-output acceptance on one audited public-safe cell. It cannot establish model preference or an Arena winner. +

+ +
+ Winner: none. Publication state: insufficient evidence. No blind comparison exists because the required visual roster is incomplete. +
+ +
+ {[ + ["Blind battles", evidence.vote_provenance.blind_battles], + ["Published ranks", evidence.publication.rank_count], + ["Human preference claims", evidence.publication.human_preference_claims ? 1 : 0], + ].map(([term, value]) => ( +
+
{term}
+
{value}
+
+ ))} +
+
+
+ +
+
+

Deterministic task success

+

Five outcomes; no preference ordering

+

+ {evidence.task_success.separation_statement} +

+ +
+ {evidence.task_success.results.map((result) => ( +
+
+
+

{result.display_name}

+ {result.baseline_id} +
+ + {result.terminal_status.replace("_", " ")} + +
+

{result.reason}

+
+
+
Scientific treatment
+
{result.task_outcome.replaceAll("_", " ")}
+
+
+
Preference eligible
+
No
+
+ {result.run_id && ( +
+
Run ID
+
{result.run_id}
+
+ )} + {result.failure_code && ( +
+
Failure code
+
{result.failure_code}
+
+ )} + {result.latency_ms !== null && ( +
+
Elapsed
+
{(result.latency_ms / 1000).toFixed(3)}s
+
+ )} +
+
+ ))} +
+
+
+ +
+
+

Complete-output acceptance

+

Two outputs passed every acceptance gate

+
+ + + + + {['Candidate', 'Output', 'Decode', 'Opening black', 'A/V drift', 'Output SHA-256', 'Acceptance SHA-256'].map((heading) => ( + + ))} + + + + {completedOutputs.map((result) => { + const output = result.output!; + return ( + + + + + + + + + + ); + })} + +
Accepted Phase 1 outputs
{heading}
{result.display_name}{(output.duration_ms / 1000).toFixed(3)}s
{output.bytes.toLocaleString()} bytes
{output.video_codec}/{output.audio_codec} · {output.format}		PASS{output.opening_black_ms}ms{output.av_start_drift_ms}ms start
{output.av_end_drift_ms}ms end		{output.output_sha256}{output.render_acceptance_sha256}
+
+
+
+ +
+
+

Disclosed diagnostic limitation

+

HyperFrames compatibility finding

+
+

{evidence.diagnostic_limitations.finding}

+
+
HyperFrames
{evidence.diagnostic_limitations.hyperframes_version}
+
Control diagnostics
{evidence.diagnostic_limitations.mechanical_control_diagnostic_count}
+
Grok diagnostics
{evidence.diagnostic_limitations.grok_4_5_diagnostic_count}
+
+

Treatment: {evidence.diagnostic_limitations.compatibility_treatment}

+

Excluded auxiliary output: {evidence.diagnostic_limitations.auxiliary_description_step}

+
+
+
+ +
+
+

Frozen provenance

+

Exact contract, runner, protocol, and output hashes

+
+ {[ + ["Arena contract", evidence.arena_binding.arena_contract_sha256], + ["Parent contract", evidence.arena_binding.parent_contract_sha256], + ["Arena source head", evidence.arena_binding.source_head_sha], + ["Runner head", evidence.frozen_evidence.runner.head_sha], + ["Runner protocol", evidence.frozen_evidence.runner.protocol_sha256], + ["Runner prompt", evidence.frozen_evidence.runner.prompt_sha256], + ["Tool registry", evidence.frozen_evidence.tool_registry_sha256], + ["Case pack", evidence.frozen_evidence.case_pack_sha256], + ].map(([label, value]) => ( +
+

{label}

+ {value} +
+ ))} +
+ +
+ {[ + ["Evidence JSON", `${ARTIFACT_ROOT}/phase-1-results.json`], + ["Evidence schema", `${ARTIFACT_ROOT}/phase-1-results.schema.json`], + ["Checksums", `${ARTIFACT_ROOT}/checksums.json`], + ["Public README", `${ARTIFACT_ROOT}/README.md`], + ].map(([label, href]) => ( + + {label} + + ))} +
+

+ Evidence artifact SHA-256: {checksums.files["phase-1-results.json"]} · Next gate: {evidence.next_gate} +

+
+
+
+ ); +} diff --git a/src/app/bench/composer-arena/page.tsx b/src/app/bench/composer-arena/page.tsx index 5a55164..7d0a305 100644 --- a/src/app/bench/composer-arena/page.tsx +++ b/src/app/bench/composer-arena/page.tsx @@ -73,11 +73,11 @@ export default async function ComposerArenaPage() { Composer Archive-to-Output Arena v0

- A publication-candidate surface for comparing evidence-grounded editing agents and complete Composer systems. Every visible result is a synthetic UI fixture until governed real outputs replace the artifacts below. + A publication-candidate surface for comparing evidence-grounded editing agents and complete Composer systems. Leaderboard-like rows remain synthetic preview examples; audited Phase 1 task evidence is published separately as a dated pilot note.

- Official ranking is disabled. No model or media call was made for this page, and local replay judgments are excluded from all standings. + Official ranking is disabled. The preview table and replay are synthetic, local replay judgments are excluded, and the separate Phase 1 evidence note contains no winner or preference claim.
@@ -96,6 +96,28 @@ export default async function ComposerArenaPage() {
+
+
+
+
+ + evidence / pilot · not a leaderboard + +

Phase 1 deterministic evidence is available

+

+ Mechanical control and Grok 4.5 produced complete passing outputs; Sol recorded a tool-policy task failure, Kimi K3 was excluded for runtime/provider compatibility, and GLM 5.2 direct vision is not applicable. Zero blind battles, zero ranks, and zero human preference claims. +

+
+ + Open dated evidence + +
+
+
+
@@ -176,19 +198,22 @@ export default async function ComposerArenaPage() {

System-eval contract SHA-256

- {release.dataset.contract_sha256} + {release.arena_binding.arena_contract_sha256}
-

FineVideo release SHA-256

- {release.dataset.finevideo_release_sha256} +

Parent system-eval contract SHA-256

+ {release.arena_binding.parent_contract_sha256}
-

Public preview release SHA-256

- {checksums.files["release.preview.json"]} +

FineVideo release SHA-256

+ {release.dataset.finevideo_release_sha256}
+

+ Public preview release SHA-256: {checksums.files["release.preview.json"]} · Arena source head: {release.arena_binding.source_head_sha} +

- The command-room FineVideo selection and release marker pass for 24 cells. The rankable runner/runtime has not produced real model records, so these hashes establish source provenance only—not measured performance or an official ranking. + The command-room FineVideo selection and release marker pass for 24 cells. Phase 1 produced deterministic pilot evidence, but not a complete comparable seven-family roster or blind human review, so no official ranking exists.

diff --git a/src/app/bench/composer-arena/types.ts b/src/app/bench/composer-arena/types.ts index 3a5ceeb..1e9acfa 100644 --- a/src/app/bench/composer-arena/types.ts +++ b/src/app/bench/composer-arena/types.ts @@ -1,6 +1,6 @@ export type ArenaTrackId = "controlled-agent" | "end-to-end-system"; -export type ArenaStanding = "ranked" | "tie" | "insufficient_evidence"; +export type ArenaStanding = "ranked" | "tied" | "insufficient_evidence"; export type TokenUsage = { input: number; @@ -36,6 +36,7 @@ export type ArenaRow = { export type ArenaTrack = { id: ArenaTrackId; + canonical_track_id: "controlled-semantic-text-evidence-v0" | "end-to-end-visual-orchestration-v0"; title: string; description: string; rows: ArenaRow[]; @@ -56,16 +57,34 @@ export type ArenaRelease = { release_id: string; generated_at: string; release_mode: "synthetic_non_rankable_preview" | "governed_rankable_release"; + data_class: "rankable" | "exploratory" | "synthetic"; official_ranking_enabled: boolean; official_vote_backend: "not_configured" | "governed_v1"; display_label: string; + arena_binding: { + arena_id: "composer-archive-to-output-arena-v0"; + source_head_sha: string; + arena_contract_sha256: string; + parent_contract_sha256: string; + }; + publication: { + publishable: boolean; + state: "ranked" | "tied" | "insufficient_evidence"; + scope: "independent_track_results_only"; + winner: string | null; + rank_count: number; + }; + vote_provenance: { + collection_mode: "bounded_human_review" | "exploratory_model_review" | "synthetic_fixture" | "no_eligible_votes"; + crowdsourced: false; + real_human_votes: number; + }; dataset: { repo_id: string; revision: string; license_claim: string; license_scope: string; attribution: string; - contract_sha256: string; finevideo_release_sha256: string; selection_manifest_sha256: string; public_eval_source_family_count: number; @@ -77,8 +96,11 @@ export type ArenaRelease = { }; }; evaluation_policy: { + ranking_method: "Bradley-Terry logistic maximum likelihood"; confidence_level: 0.95; - minimum_battles_for_standing: number; + bootstrap_replicates: 2000; + minimum_resolved_battles_per_pair_overall: 4; + position_bias_decisive_presentations: 20; tie_rule: string; insufficient_evidence_rule: string; local_vote_policy: string; diff --git a/src/app/bench/page.tsx b/src/app/bench/page.tsx index 39747f0..20d0db1 100644 --- a/src/app/bench/page.tsx +++ b/src/app/bench/page.tsx @@ -139,7 +139,7 @@ export default function BenchPage() {

Archive understanding to reviewable output

- Synthetic, schema-driven standings, track-separated metrics, non-rankable capability cards, and anonymous A/B replay. No official vote backend or measured model result is active. + Synthetic preview examples, track-separated contracts, and anonymous A/B replay sit beside a dated Phase 1 deterministic-evidence note. There is no official ranking, winner, or human preference claim.

Date: Sun, 19 Jul 2026 07:29:42 -0700 Subject: [PATCH 3/4] fix: publish sealed seven-family evidence --- .../evidence/pilot/2026-07-19/README.md | 21 +- .../evidence/pilot/2026-07-19/checksums.json | 6 +- .../pilot/2026-07-19/phase-1-results.json | 161 ----------- .../2026-07-19/phase-1-results.schema.json | 165 ------------ .../2026-07-19/seven-family-results.json | 93 +++++++ .../seven-family-results.schema.json | 150 +++++++++++ .../composer-arena/v0/battles.preview.jsonl | 4 +- public/bench/composer-arena/v0/checksums.json | 10 +- public/bench/composer-arena/v0/handoff.md | 15 +- public/bench/composer-arena/v0/methodology.md | 25 +- .../composer-arena/v0/release.preview.json | 160 ++--------- .../composer-arena/v0/release.schema.json | 2 +- scripts/check-composer-arena.mjs | 250 ++++++++++++------ src/app/bench/composer-arena/arena-client.tsx | 32 ++- .../evidence/pilot/2026-07-19/data.ts | 90 +++---- .../evidence/pilot/2026-07-19/page.tsx | 186 ++++++------- src/app/bench/composer-arena/page.tsx | 16 +- src/app/bench/page.tsx | 4 +- 18 files changed, 606 insertions(+), 784 deletions(-) delete mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json delete mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json create mode 100644 public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md index 10d4f9e..9329ffe 100644 --- a/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md @@ -1,17 +1,14 @@ -# Composer Arena Phase 1 evidence pilot — 2026-07-19 +# Composer Arena seven-family task-success evidence — 2026-07-19 -This is a sanitized, dated deterministic-evidence packet. It is not a leaderboard, preference result, or production publication approval. +This is a sanitized deterministic-evidence aggregate over seven eligible FineVideo public-eval cells from seven distinct source families. It is not a leaderboard, preference result, or publication approval. -The packet is bound to Composer Arena contract `composer-archive-to-output-arena-v0`, source head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7`, and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. +The public packet reports only cohort counts and stable evidence hashes: -It reports task outcomes and complete-output acceptance only: +- mechanical control: 6/7 structural passes and 5/7 complete outputs accepted; +- evaluated lane A: 2/7 complete outputs accepted; +- evaluated lane B: 4/7 complete outputs accepted; +- zero blind battles, votes, ranks, or winner. -- mechanical control: pass and complete output; -- Grok 4.5: pass and complete output; -- Sol: `tool_denied` policy/task failure with no rendered candidate; -- Kimi K3: runtime/provider compatibility exclusion, not editing-quality loss; -- GLM 5.2: direct visual perception not applicable, not a failure. +Evaluated model identities remain sealed pending the founder's blind vote. The real blind packet, candidate media, per-cell identifiers, output hashes, prompts, run identifiers, private paths, and identity mapping are not included here. -There are zero blind battles, zero human preference votes, zero ranks, and no winner. The required visual roster is incomplete, so the upstream Arena contract permits no Bradley-Terry observation. - -The public JSON contains no local/founder paths, media or source URLs, customer/GMA data, credentials, hidden-test material, private case annotations, or raw reasoning traces. +Deterministic task success and complete-output acceptance do not measure editorial preference. SEA Broadcast ASR remains a separate benchmark suite with independent sources, rights, scoring, and publication gates. diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json index bd1cfb2..2c50d72 100644 --- a/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json @@ -1,8 +1,8 @@ { "algorithm": "sha256", "files": { - "phase-1-results.json": "8525a64708f4da63ae0cf6dab3be5efa4786ad3466cbddf25ad6a462a0c0508a", - "phase-1-results.schema.json": "14d5ea76c9dbf2c9a6cc1fb8ece8659a3f069f3130f0bbcf4ffebc1a26e00f24", - "README.md": "08d34fe486b2956eec990c956204ad84a9ce362e9c0ddf03a89628017a34052f" + "seven-family-results.json": "ed2cba9fb604a7d2db70b862fd760c5287e55b21d3b4cc6aeb8415285fa57d1f", + "seven-family-results.schema.json": "a26f151cb7102294f8e7a0712031bd6a2462b1561568901edc37f97e586bade3", + "README.md": "0630d7415cded4039c6fde74101610a6b746cd0556fa78807c5e2fff51b6a61a" } } diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json deleted file mode 100644 index 6df2020..0000000 --- a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "schema_version": 1, - "evidence_id": "composer-arena-phase-1-pilot-2026-07-19", - "evidence_date": "2026-07-19", - "status": "phase-1-complete-publication-not-approved", - "data_class": "pilot_evidence", - "arena_binding": { - "arena_id": "composer-archive-to-output-arena-v0", - "source_head_sha": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", - "arena_contract_sha256": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", - "parent_contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" - }, - "publication": { - "surface": "evidence/pilot", - "publishable": false, - "state": "insufficient_evidence", - "leaderboard": false, - "winner": null, - "rank_count": 0, - "human_preference_claims": false, - "reasons": [ - "Only one complete model output exists for the audited cell.", - "The required visual roster is incomplete and therefore generates no preference battle.", - "The seven immutable holdout source families have not completed the frozen protocol." - ] - }, - "vote_provenance": { - "collection_mode": "no_eligible_votes", - "crowdsourced": false, - "planned_battle_groups": 0, - "blind_battles": 0, - "real_human_votes": 0, - "resolved_battle_groups": 0 - }, - "frozen_evidence": { - "dataset_repo_id": "HuggingFaceFV/finevideo", - "dataset_revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", - "selection_sha256": "39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4", - "release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", - "case_pack_sha256": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203", - "content_evidence_sha256": "7f798fce08b0a2a2bf5d24e49f28f306bae87e0994b947d6c593ac2cdbba2128", - "task_brief_sha256": "45c58fca279847ce3029166a38814cfd4b13065bac93a6ddc71f24e480043569", - "tool_registry_sha256": "2f3a866c9ff9debc590f30361d6fabdbef0b808ded69beddd2054eac5795975c", - "runner": { - "version": "composer-system-eval-runner-v2", - "head_sha": "97d2606c98b2c38d3fbb0e338fbbebc2c6e8b22d", - "protocol_sha256": "5965228c9b0bbf0d8d46667b951d750f1dc1ffe9b5a3bb39b36fddbb39592d1b", - "prompt_sha256": "0278214adab1aea4c25681f64bcc66a8563910f51250cf6fbf04263faa563c37" - } - }, - "task_success": { - "separation_statement": "Deterministic task outcomes are evidence only and never enter a preference score.", - "results": [ - { - "baseline_id": "chronological-mechanical-v1", - "display_name": "Mechanical control", - "classification": "control", - "terminal_status": "passed", - "task_outcome": "pass", - "preference_eligible": false, - "quality_loss": false, - "run_id": "control-a4af853df6a1e23c6b10cfa9a84abcc4", - "latency_ms": null, - "failure_code": null, - "reason": "Zero-model chronological control completed the frozen task and output acceptance gates.", - "output": { - "complete": true, - "duration_ms": 45013, - "bytes": 22942811, - "video_codec": "H.264", - "audio_codec": "AAC", - "format": "1080p30", - "decode_passed": true, - "opening_black_ms": 0, - "av_start_drift_ms": 0, - "av_end_drift_ms": 13.333, - "output_sha256": "cb7a918b3c3f2b1eb44b944fa6d6a453cc71cd020e806274f976cf0a2901a2c4", - "render_acceptance_sha256": "279ee22dadbb53d861af7da139af4008e2b3bfd58da6fecb629b8b2f9bf0c6ad" - } - }, - { - "baseline_id": "grok-4.5-medium-v1", - "display_name": "Grok 4.5 medium", - "classification": "evaluated_model", - "terminal_status": "passed", - "task_outcome": "pass", - "preference_eligible": false, - "quality_loss": false, - "run_id": "rankable-4771baa518add8e98618bf579eac5976", - "latency_ms": 291034, - "failure_code": null, - "reason": "Complete output passed deterministic and render acceptance, but no second comparable roster output exists for a blind battle.", - "output": { - "complete": true, - "duration_ms": 58539, - "bytes": 27860030, - "video_codec": "H.264", - "audio_codec": "AAC", - "format": "1080p30", - "decode_passed": true, - "opening_black_ms": 0, - "av_start_drift_ms": 0, - "av_end_drift_ms": 5.334, - "output_sha256": "d8068df3d87c7e2cba6e609d098918e0cb705e9ac69cb6f2c45f23cd2000b671", - "render_acceptance_sha256": "607a2ce5da7b4a2cbdc81d3208a552d2164f7cd8775cc8d2c93ad53f25db878d" - } - }, - { - "baseline_id": "sol-medium-v1", - "display_name": "Sol medium", - "classification": "evaluated_model", - "terminal_status": "failed", - "task_outcome": "policy_task_failure", - "preference_eligible": false, - "quality_loss": false, - "run_id": "rankable-7053fc4cd2bbab5e6cd4ec5636ecd10e", - "latency_ms": 233774, - "failure_code": "tool_denied", - "reason": "The run attempted a direct timing-HTML mutation prohibited by the frozen runner-v2 tool policy; no rendered candidate or preference battle exists.", - "output": null - }, - { - "baseline_id": "kimi-k3-high-v1", - "display_name": "Kimi K3 high", - "classification": "evaluated_model", - "terminal_status": "excluded", - "task_outcome": "runtime_provider_compatibility", - "preference_eligible": false, - "quality_loss": false, - "run_id": "rankable-c00071c662b6192a996c161a10e0e62d", - "latency_ms": 2168, - "failure_code": null, - "reason": "Excluded at the runtime/provider compatibility boundary with zero tool calls; this is not an editing-quality loss.", - "output": null - }, - { - "baseline_id": "glm-5.2-high-v1", - "display_name": "GLM 5.2 high", - "classification": "not_applicable", - "terminal_status": "not_applicable", - "task_outcome": "direct_perception_unavailable", - "preference_eligible": false, - "quality_loss": false, - "run_id": null, - "latency_ms": null, - "failure_code": null, - "reason": "Direct visual perception is unavailable and GLM 5.2 remains outside the visual preference roster; this is not a failure.", - "output": null - } - ] - }, - "diagnostic_limitations": { - "hyperframes_version": "0.7.64", - "finding": "The renderer emitted non-blocking data-end/data-duration static-guard diagnostics even though browser, render, decode, duration, and A/V acceptance gates passed.", - "mechanical_control_diagnostic_count": 8, - "grok_4_5_diagnostic_count": 6, - "compatibility_treatment": "Preserve as a Composer/HyperFrames compatibility finding; do not reinterpret the passing outputs as failed.", - "auxiliary_description_step": "A snapshot command invoked an auxiliary Gemini description step unexpectedly. That output is excluded from evidence, scoring, and judgment." - }, - "next_gate": "all-seven-immutable-holdout-families" -} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json deleted file mode 100644 index ae3e76f..0000000 --- a/public/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json +++ /dev/null @@ -1,165 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://agentic.video/bench/composer-arena/evidence/pilot/2026-07-19/phase-1-results.schema.json", - "title": "Composer Arena Phase 1 public pilot evidence", - "type": "object", - "additionalProperties": false, - "required": [ - "schema_version", - "evidence_id", - "evidence_date", - "status", - "data_class", - "arena_binding", - "publication", - "vote_provenance", - "frozen_evidence", - "task_success", - "diagnostic_limitations", - "next_gate" - ], - "properties": { - "schema_version": { "const": 1 }, - "evidence_id": { "const": "composer-arena-phase-1-pilot-2026-07-19" }, - "evidence_date": { "const": "2026-07-19" }, - "status": { "const": "phase-1-complete-publication-not-approved" }, - "data_class": { "const": "pilot_evidence" }, - "arena_binding": { - "type": "object", - "additionalProperties": false, - "required": ["arena_id", "source_head_sha", "arena_contract_sha256", "parent_contract_sha256"], - "properties": { - "arena_id": { "const": "composer-archive-to-output-arena-v0" }, - "source_head_sha": { "const": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7" }, - "arena_contract_sha256": { "const": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5" }, - "parent_contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" } - } - }, - "publication": { - "type": "object", - "additionalProperties": false, - "required": ["surface", "publishable", "state", "leaderboard", "winner", "rank_count", "human_preference_claims", "reasons"], - "properties": { - "surface": { "const": "evidence/pilot" }, - "publishable": { "const": false }, - "state": { "const": "insufficient_evidence" }, - "leaderboard": { "const": false }, - "winner": { "type": "null" }, - "rank_count": { "const": 0 }, - "human_preference_claims": { "const": false }, - "reasons": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } } - } - }, - "vote_provenance": { - "type": "object", - "additionalProperties": false, - "required": ["collection_mode", "crowdsourced", "planned_battle_groups", "blind_battles", "real_human_votes", "resolved_battle_groups"], - "properties": { - "collection_mode": { "const": "no_eligible_votes" }, - "crowdsourced": { "const": false }, - "planned_battle_groups": { "const": 0 }, - "blind_battles": { "const": 0 }, - "real_human_votes": { "const": 0 }, - "resolved_battle_groups": { "const": 0 } - } - }, - "frozen_evidence": { "$ref": "#/$defs/frozenEvidence" }, - "task_success": { - "type": "object", - "additionalProperties": false, - "required": ["separation_statement", "results"], - "properties": { - "separation_statement": { "type": "string", "minLength": 1 }, - "results": { - "type": "array", - "minItems": 5, - "maxItems": 5, - "items": { "$ref": "#/$defs/taskResult" } - } - } - }, - "diagnostic_limitations": { "$ref": "#/$defs/diagnosticLimitations" }, - "next_gate": { "const": "all-seven-immutable-holdout-families" } - }, - "$defs": { - "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, - "sha1": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, - "frozenEvidence": { - "type": "object", - "additionalProperties": false, - "required": ["dataset_repo_id", "dataset_revision", "selection_sha256", "release_sha256", "case_pack_sha256", "content_evidence_sha256", "task_brief_sha256", "tool_registry_sha256", "runner"], - "properties": { - "dataset_repo_id": { "const": "HuggingFaceFV/finevideo" }, - "dataset_revision": { "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" }, - "selection_sha256": { "$ref": "#/$defs/sha256" }, - "release_sha256": { "$ref": "#/$defs/sha256" }, - "case_pack_sha256": { "$ref": "#/$defs/sha256" }, - "content_evidence_sha256": { "$ref": "#/$defs/sha256" }, - "task_brief_sha256": { "$ref": "#/$defs/sha256" }, - "tool_registry_sha256": { "$ref": "#/$defs/sha256" }, - "runner": { - "type": "object", - "additionalProperties": false, - "required": ["version", "head_sha", "protocol_sha256", "prompt_sha256"], - "properties": { - "version": { "const": "composer-system-eval-runner-v2" }, - "head_sha": { "$ref": "#/$defs/sha1" }, - "protocol_sha256": { "$ref": "#/$defs/sha256" }, - "prompt_sha256": { "$ref": "#/$defs/sha256" } - } - } - } - }, - "taskResult": { - "type": "object", - "additionalProperties": false, - "required": ["baseline_id", "display_name", "classification", "terminal_status", "task_outcome", "preference_eligible", "quality_loss", "run_id", "latency_ms", "failure_code", "reason", "output"], - "properties": { - "baseline_id": { "enum": ["chronological-mechanical-v1", "grok-4.5-medium-v1", "sol-medium-v1", "kimi-k3-high-v1", "glm-5.2-high-v1"] }, - "display_name": { "type": "string", "minLength": 1 }, - "classification": { "enum": ["control", "evaluated_model", "not_applicable"] }, - "terminal_status": { "enum": ["passed", "failed", "excluded", "not_applicable"] }, - "task_outcome": { "enum": ["pass", "policy_task_failure", "runtime_provider_compatibility", "direct_perception_unavailable"] }, - "preference_eligible": { "const": false }, - "quality_loss": { "const": false }, - "run_id": { "type": ["string", "null"] }, - "latency_ms": { "type": ["integer", "null"], "minimum": 0 }, - "failure_code": { "type": ["string", "null"] }, - "reason": { "type": "string", "minLength": 1 }, - "output": { "oneOf": [{ "$ref": "#/$defs/output" }, { "type": "null" }] } - } - }, - "output": { - "type": "object", - "additionalProperties": false, - "required": ["complete", "duration_ms", "bytes", "video_codec", "audio_codec", "format", "decode_passed", "opening_black_ms", "av_start_drift_ms", "av_end_drift_ms", "output_sha256", "render_acceptance_sha256"], - "properties": { - "complete": { "const": true }, - "duration_ms": { "type": "integer", "minimum": 1 }, - "bytes": { "type": "integer", "minimum": 1 }, - "video_codec": { "type": "string", "minLength": 1 }, - "audio_codec": { "type": "string", "minLength": 1 }, - "format": { "type": "string", "minLength": 1 }, - "decode_passed": { "const": true }, - "opening_black_ms": { "type": "number", "minimum": 0 }, - "av_start_drift_ms": { "type": "number", "minimum": 0 }, - "av_end_drift_ms": { "type": "number", "minimum": 0 }, - "output_sha256": { "$ref": "#/$defs/sha256" }, - "render_acceptance_sha256": { "$ref": "#/$defs/sha256" } - } - }, - "diagnosticLimitations": { - "type": "object", - "additionalProperties": false, - "required": ["hyperframes_version", "finding", "mechanical_control_diagnostic_count", "grok_4_5_diagnostic_count", "compatibility_treatment", "auxiliary_description_step"], - "properties": { - "hyperframes_version": { "const": "0.7.64" }, - "finding": { "type": "string", "minLength": 1 }, - "mechanical_control_diagnostic_count": { "const": 8 }, - "grok_4_5_diagnostic_count": { "const": 6 }, - "compatibility_treatment": { "type": "string", "minLength": 1 }, - "auxiliary_description_step": { "type": "string", "minLength": 1 } - } - } - } -} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json new file mode 100644 index 0000000..4ae7149 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json @@ -0,0 +1,93 @@ +{ + "schema_version": 1, + "evidence_id": "composer-arena-seven-family-task-success-2026-07-19", + "evidence_date": "2026-07-19", + "status": "seven-family-terminal-task-success-publication-not-approved", + "data_class": "sanitized_task_success_aggregate", + "publication": { + "surface": "evidence/pilot", + "publishable": false, + "state": "insufficient_evidence", + "leaderboard": false, + "winner": null, + "rank_count": 0, + "human_preference_claims": false + }, + "vote_provenance": { + "collection_mode": "no_eligible_votes", + "crowdsourced": false, + "blind_battles": 0, + "real_human_votes": 0, + "resolved_battle_groups": 0 + }, + "frozen_evidence": { + "dataset_repo_id": "HuggingFaceFV/finevideo", + "dataset_revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", + "staging_manifest_sha256": "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8", + "contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", + "release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + "case_pack_sha256": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203" + }, + "cohort": { + "eligible_cells": 7, + "distinct_source_families": 7, + "favorable_selection": false, + "source_family_disjoint": true + }, + "task_success": { + "separation_statement": "Deterministic task success and complete-output acceptance are system evidence only; they never create a preference score, rank, or winner.", + "lanes": [ + { + "lane_id": "mechanical-control", + "classification": "control", + "identity_state": "public_control", + "attempted_cells": 7, + "structural_passes": 6, + "structural_failures": 1, + "complete_outputs_accepted": 5, + "complete_output_failures": 2, + "model_calls": 0, + "source_summary_sha256": "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57" + }, + { + "lane_id": "evaluated-lane-a", + "classification": "evaluated_system", + "identity_state": "sealed_pending_founder_vote", + "attempted_cells": 7, + "structural_passes": 2, + "structural_failures": 5, + "complete_outputs_accepted": 2, + "complete_output_failures": 5, + "model_calls": 7, + "source_summary_sha256": "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a" + }, + { + "lane_id": "evaluated-lane-b", + "classification": "evaluated_system", + "identity_state": "sealed_pending_founder_vote", + "attempted_cells": 7, + "structural_passes": 4, + "structural_failures": 3, + "complete_outputs_accepted": 4, + "complete_output_failures": 3, + "model_calls": 7, + "source_summary_sha256": "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456" + } + ] + }, + "blind_review": { + "packet_public": false, + "candidate_outputs_public": false, + "identity_mapping_public": false, + "founder_vote_recorded": false, + "next_gate": "founder_blind_vote_then_separate_identity_reveal" + }, + "limitations": [ + "Evaluated lane identities are deliberately sealed until the founder records the blind vote.", + "The public aggregate contains no per-cell identifiers, output hashes, candidate media, prompts, run identifiers, or identity mapping.", + "Complete-output acceptance measures deterministic delivery gates, not editorial quality or human preference.", + "The controlled-agent track remains blocked because the released evidence pack contains no transcript, OCR, or semantic events.", + "One mechanical-control cell failed structurally and one rendered control output failed the opening-black gate; neither failure was repaired inside the frozen cohort.", + "No official ranking, winner, governed vote backend, or public blind packet exists." + ] +} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json new file mode 100644 index 0000000..8942372 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json @@ -0,0 +1,150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json", + "title": "Composer Arena seven-family public task-success evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "evidence_id", + "evidence_date", + "status", + "data_class", + "publication", + "vote_provenance", + "frozen_evidence", + "cohort", + "task_success", + "blind_review", + "limitations" + ], + "properties": { + "schema_version": { "const": 1 }, + "evidence_id": { "const": "composer-arena-seven-family-task-success-2026-07-19" }, + "evidence_date": { "const": "2026-07-19" }, + "status": { "const": "seven-family-terminal-task-success-publication-not-approved" }, + "data_class": { "const": "sanitized_task_success_aggregate" }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["surface", "publishable", "state", "leaderboard", "winner", "rank_count", "human_preference_claims"], + "properties": { + "surface": { "const": "evidence/pilot" }, + "publishable": { "const": false }, + "state": { "const": "insufficient_evidence" }, + "leaderboard": { "const": false }, + "winner": { "type": "null" }, + "rank_count": { "const": 0 }, + "human_preference_claims": { "const": false } + } + }, + "vote_provenance": { + "type": "object", + "additionalProperties": false, + "required": ["collection_mode", "crowdsourced", "blind_battles", "real_human_votes", "resolved_battle_groups"], + "properties": { + "collection_mode": { "const": "no_eligible_votes" }, + "crowdsourced": { "const": false }, + "blind_battles": { "const": 0 }, + "real_human_votes": { "const": 0 }, + "resolved_battle_groups": { "const": 0 } + } + }, + "frozen_evidence": { + "type": "object", + "additionalProperties": false, + "required": ["dataset_repo_id", "dataset_revision", "staging_manifest_sha256", "contract_sha256", "release_sha256", "case_pack_sha256"], + "properties": { + "dataset_repo_id": { "const": "HuggingFaceFV/finevideo" }, + "dataset_revision": { "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" }, + "staging_manifest_sha256": { "const": "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8" }, + "contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" }, + "release_sha256": { "const": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926" }, + "case_pack_sha256": { "const": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203" } + } + }, + "cohort": { + "type": "object", + "additionalProperties": false, + "required": ["eligible_cells", "distinct_source_families", "favorable_selection", "source_family_disjoint"], + "properties": { + "eligible_cells": { "const": 7 }, + "distinct_source_families": { "const": 7 }, + "favorable_selection": { "const": false }, + "source_family_disjoint": { "const": true } + } + }, + "task_success": { + "type": "object", + "additionalProperties": false, + "required": ["separation_statement", "lanes"], + "properties": { + "separation_statement": { "type": "string", "minLength": 1 }, + "lanes": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + { "$ref": "#/$defs/controlLane" }, + { "$ref": "#/$defs/evaluatedLaneA" }, + { "$ref": "#/$defs/evaluatedLaneB" } + ] + } + } + }, + "blind_review": { + "type": "object", + "additionalProperties": false, + "required": ["packet_public", "candidate_outputs_public", "identity_mapping_public", "founder_vote_recorded", "next_gate"], + "properties": { + "packet_public": { "const": false }, + "candidate_outputs_public": { "const": false }, + "identity_mapping_public": { "const": false }, + "founder_vote_recorded": { "const": false }, + "next_gate": { "const": "founder_blind_vote_then_separate_identity_reveal" } + } + }, + "limitations": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + }, + "$defs": { + "laneBase": { + "type": "object", + "additionalProperties": false, + "required": ["lane_id", "classification", "identity_state", "attempted_cells", "structural_passes", "structural_failures", "complete_outputs_accepted", "complete_output_failures", "model_calls", "source_summary_sha256"], + "properties": { + "lane_id": { "type": "string", "minLength": 1 }, + "classification": { "enum": ["control", "evaluated_system"] }, + "identity_state": { "enum": ["public_control", "sealed_pending_founder_vote"] }, + "attempted_cells": { "const": 7 }, + "structural_passes": { "type": "integer", "minimum": 0, "maximum": 7 }, + "structural_failures": { "type": "integer", "minimum": 0, "maximum": 7 }, + "complete_outputs_accepted": { "type": "integer", "minimum": 0, "maximum": 7 }, + "complete_output_failures": { "type": "integer", "minimum": 0, "maximum": 7 }, + "model_calls": { "type": "integer", "minimum": 0 }, + "source_summary_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + "controlLane": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "mechanical-control" }, "classification": { "const": "control" }, "identity_state": { "const": "public_control" }, "structural_passes": { "const": 6 }, "structural_failures": { "const": 1 }, "complete_outputs_accepted": { "const": 5 }, "complete_output_failures": { "const": 2 }, "model_calls": { "const": 0 }, "source_summary_sha256": { "const": "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57" } } } + ] + }, + "evaluatedLaneA": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "evaluated-lane-a" }, "classification": { "const": "evaluated_system" }, "identity_state": { "const": "sealed_pending_founder_vote" }, "structural_passes": { "const": 2 }, "structural_failures": { "const": 5 }, "complete_outputs_accepted": { "const": 2 }, "complete_output_failures": { "const": 5 }, "model_calls": { "const": 7 }, "source_summary_sha256": { "const": "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a" } } } + ] + }, + "evaluatedLaneB": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "evaluated-lane-b" }, "classification": { "const": "evaluated_system" }, "identity_state": { "const": "sealed_pending_founder_vote" }, "structural_passes": { "const": 4 }, "structural_failures": { "const": 3 }, "complete_outputs_accepted": { "const": 4 }, "complete_output_failures": { "const": 3 }, "model_calls": { "const": 7 }, "source_summary_sha256": { "const": "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456" } } } + ] + } + } +} diff --git a/public/bench/composer-arena/v0/battles.preview.jsonl b/public/bench/composer-arena/v0/battles.preview.jsonl index e0020ff..6ebc7a4 100644 --- a/public/bench/composer-arena/v0/battles.preview.jsonl +++ b/public/bench/composer-arena/v0/battles.preview.jsonl @@ -1,2 +1,2 @@ -{"battle_id":"preview-controlled-001","track":"controlled-agent","task_slice":"temporal-ordering","prompt":"Create an evidence-grounded edit plan that opens on the constraint, preserves source chronology, and cites every selected window.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-chronology","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"sol-medium-v1","model":"GPT-5.6 Sol","provider":"AgenticFlow","version":"sol-medium-v1","output_title":"Output A","output_summary":"Three-window plan with explicit evidence IDs, chronological ordering, and a short unsupported transition note.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Methodology","href":"/bench/composer-arena/v0/methodology.md"}]},"right":{"candidate_id":"grok-4.5-medium-v1","model":"Grok 4.5","provider":"AgenticFlow","version":"grok-4.5-medium-v1","output_title":"Output B","output_summary":"Three-window plan with complete evidence IDs and valid ranges; the opening constraint is satisfied after a brief setup beat.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Release fixture","href":"/bench/composer-arena/v0/release.preview.json"}]}} -{"battle_id":"preview-system-001","track":"end-to-end-system","task_slice":"reviewable-delivery","prompt":"Produce a structurally valid 45-second review cut with synchronized source windows, no blocking gaps, and a provenance-complete delivery record.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-delivery","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"grok-4.5-medium-v1-composer-v0","model":"Grok 4.5 + Composer","provider":"AgenticFlow / Pixel ML","version":"grok-4.5-medium-v1 / composer-v0","output_title":"Output A","output_summary":"Valid 45-second timeline with complete provenance; one caption-safe-area warning remains non-blocking.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Schema","href":"/bench/composer-arena/v0/release.schema.json"}]},"right":{"candidate_id":"sol-medium-v1-composer-v0","model":"GPT-5.6 Sol + Composer","provider":"AgenticFlow / Pixel ML","version":"sol-medium-v1 / composer-v0","output_title":"Output B","output_summary":"Valid synchronized timeline and clean structural gate; delivery note omits the synthetic source-family label.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Handoff contract","href":"/bench/composer-arena/v0/handoff.md"}]}} +{"battle_id":"preview-controlled-001","track":"controlled-agent","task_slice":"temporal-ordering","prompt":"Create an evidence-grounded edit plan that opens on the constraint, preserves source chronology, and cites every selected window.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-chronology","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"demo-controlled-alpha","model":"Synthetic demo Alpha","provider":"Not applicable","version":"fixture-alpha-v1","output_title":"Output A","output_summary":"Three-window synthetic plan with explicit evidence IDs, chronological ordering, and a short unsupported transition note.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Methodology","href":"/bench/composer-arena/v0/methodology.md"}]},"right":{"candidate_id":"demo-controlled-beta","model":"Synthetic demo Beta","provider":"Not applicable","version":"fixture-beta-v1","output_title":"Output B","output_summary":"Three-window synthetic plan with complete evidence IDs and valid ranges; the opening constraint follows a brief setup beat.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Release fixture","href":"/bench/composer-arena/v0/release.preview.json"}]}} +{"battle_id":"preview-system-001","track":"end-to-end-system","task_slice":"reviewable-delivery","prompt":"Produce a structurally valid 45-second review cut with synchronized source windows, no blocking gaps, and a provenance-complete delivery record.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-delivery","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"demo-system-alpha","model":"Synthetic demo Alpha","provider":"Not applicable","version":"fixture-alpha-v1","output_title":"Output A","output_summary":"Synthetic 45-second timeline with complete provenance and one non-blocking safe-area warning.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Schema","href":"/bench/composer-arena/v0/release.schema.json"}]},"right":{"candidate_id":"demo-system-beta","model":"Synthetic demo Beta","provider":"Not applicable","version":"fixture-beta-v1","output_title":"Output B","output_summary":"Synthetic synchronized timeline with a clean structural gate and an omitted source-family label.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Handoff contract","href":"/bench/composer-arena/v0/handoff.md"}]}} diff --git a/public/bench/composer-arena/v0/checksums.json b/public/bench/composer-arena/v0/checksums.json index a4204a4..4da1bbe 100644 --- a/public/bench/composer-arena/v0/checksums.json +++ b/public/bench/composer-arena/v0/checksums.json @@ -1,11 +1,11 @@ { "algorithm": "sha256", "files": { - "release.schema.json": "da3cb101dec52ac3cdeab1dc9e6f041668df150857d4781676e4b1b0399ae55f", + "release.schema.json": "a42e66add183d8e35d40fce36fa0b8ab3ba2c1e4656d7192398b0873a6f62b35", "battle.schema.json": "b235183b07bf469172c79e8746275f2dd81dc30517667db5fb9a40f07c1f1572", - "release.preview.json": "0d33d0aa14ed6295c11d23e81fe985af003941fecab12131e4f3fc3eee874915", - "battles.preview.jsonl": "cb03fa3f39acc9c6488aa2beb45b08dc3cefe9fcaae236a66805873bd05d0d6c", - "methodology.md": "b989ced1c20285f6257a9d3be1d11097fd29a11f6e4def030d03c194c2a29a55", - "handoff.md": "30f58c039880905d6fa6cad884c68417fd483af7d9ef0ef31b40cac2cf7beacb" + "release.preview.json": "ae76298c693969dd3d1b2c7c687a791b715820daf4f0570a5a9711135ae202bd", + "battles.preview.jsonl": "2b99e8b7f89e073ed81daeab19cd86ec984aae87f8a3ac26bc079ae66e0cb59b", + "methodology.md": "7eda2248f9b434642f796034ca1760c40c6d485b00df17fd41766210936ae3d2", + "handoff.md": "7419838696f005a0048aaaf0e13ab35e7fe50f87abded698bc7307db3bf35a10" } } diff --git a/public/bench/composer-arena/v0/handoff.md b/public/bench/composer-arena/v0/handoff.md index ac65c46..299d4ec 100644 --- a/public/bench/composer-arena/v0/handoff.md +++ b/public/bench/composer-arena/v0/handoff.md @@ -1,6 +1,6 @@ # Composer Arena real-data handoff -The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. Real public artifacts can replace those files without changing React code when they preserve the schema and filenames below. +The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. The current release has empty standings and the battle file is a generic synthetic interaction demo. A real blind packet must not replace either public file before the founder votes and a separate identity reveal is authorized. The canonical upstream contract binding is `composer-archive-to-output-arena-v0` at source head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7`, Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`, and parent system-eval contract SHA-256 `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624`. A future adapter must fail closed if any binding drifts. @@ -11,7 +11,7 @@ The canonical upstream contract binding is `composer-archive-to-output-arena-v0` 3. `checksums.json` — raw SHA-256 values for the schema, release, battles, methodology, and this handoff. 4. `methodology.md` — frozen methodology, estimator, vote policy, source/rights boundary, contamination policy, and limitations. -Before a rankable release, change `release_mode` to `governed_rankable_release` and `official_ranking_enabled` to `true` only after the already-green FineVideo release marker is joined by real runner records, a governed vote backend when votes are included, a frozen statistical policy, and command-room review. Synthetic or local/demo data must remain `synthetic_non_rankable_preview`. +Before a rankable release, change `release_mode` to `governed_rankable_release` and `official_ranking_enabled` to `true` only after the already-green FineVideo release marker is joined by real runner records, a governed vote backend when votes are included, a frozen statistical policy, and command-room review. Synthetic or local/demo data remains `synthetic_non_rankable_preview`; sanitized task-success evidence remains separate from standings. The canonical upstream publication vocabulary is `ranked`, `tied`, and `insufficient_evidence`; public results have `scope: independent_track_results_only`. `controlled-semantic-text-evidence-v0` and `end-to-end-visual-orchestration-v0` remain separate, and a combined winner is forbidden. @@ -41,7 +41,14 @@ Each line must be a self-contained object with: - a `source` object containing fixture/cell identity, source-family ID, immutable revision, and `rankable`; - `left` and `right` candidates containing a stable candidate ID, hidden model/provider/version identity, public output title/summary, and one or more provenance/evidence links. -The browser may swap left/right display order. Judgment choices are exactly `left`, `right`, `tie`, and `both_bad`. The public client reveals identity only after a judgment and sends no vote request. +The browser may swap left/right display order. Local demo choices are exactly `left`, `right`, `tie`, `both_bad`, and `abstain`. The public client reveals only generic demo identities after a choice and sends no vote request. + +## Private blind packet boundary + +- Keep the real packet, candidate media, per-cell identifiers, output hashes, prompts, run identifiers, and model mapping outside `public/` until the founder records a vote. +- Public pre-vote evidence may contain only aggregate task-success counts, shared-input hashes, and identity-sealed source-summary commitments. +- Do not expose a real candidate through static JSON, HTML source, build output, evidence links, checksums, or downloadable artifacts. +- Publish any model-identity reveal as a separate, dated post-vote artifact so the pre-vote evidence remains immutable. ## Promotion gates @@ -52,6 +59,6 @@ The browser may swap left/right display order. Judgment choices are exactly `lef - No hidden test data, customer/GMA data, credentials, signed URLs, or raw reasoning traces enter public artifacts. - Every model route passes exact-provider/model/version preflight with no fallback substitution. - The statistical exporter emits ties and insufficient-evidence states rather than forcing ranks. -- Evidence/smoke partitions and pilot failures create deterministic evidence only; an incomplete required-roster stratum creates zero battles and no Bradley-Terry observation. +- Deterministic seven-family task-success evidence creates no battle or Bradley-Terry observation without a private comparable packet and a governed human judgment. - Official votes, if any, come only from a governed backend with a versioned inclusion policy. - Schema validation, checksum verification, lint, build, route contract checks, and HTTP/UI smoke are green. diff --git a/public/bench/composer-arena/v0/methodology.md b/public/bench/composer-arena/v0/methodology.md index a2db4c0..48f6319 100644 --- a/public/bench/composer-arena/v0/methodology.md +++ b/public/bench/composer-arena/v0/methodology.md @@ -1,24 +1,26 @@ # Composer Archive-to-Output Arena v0 methodology -Status: **non-rankable preview**. The checked-in metrics and battle outputs are synthetic interface fixtures, not measured model results. This public adapter is bound to `PixelML/agentic_video_intelligence` head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. +Status: **non-rankable preview**. The public standings are empty. The dated evidence page contains sanitized seven-family task-success counts, while the A/B replay contains generic synthetic interaction fixtures only. This public adapter is bound to `PixelML/agentic_video_intelligence` head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. ## Tracks -- `controlled-semantic-text-evidence-v0` / `controlled-agent` requires all four systems over one genuine, hash-identical transcript/OCR/semantic-event artifact. That artifact is unavailable, so the real v0 track is blocked and the visible rows are synthetic interface examples only. -- `end-to-end-visual-orchestration-v0` / `end-to-end-system` requires Sol, Grok 4.5, and Kimi K3 over the same visual case pack. GLM 5.2 direct perception is explicitly not applicable and remains outside that roster. +- `controlled-semantic-text-evidence-v0` / `controlled-agent` requires one genuine, hash-identical transcript/OCR/semantic-event artifact. That artifact is unavailable, so the track remains blocked before model execution. +- `end-to-end-visual-orchestration-v0` / `end-to-end-system` has terminal seven-family deterministic task-success evidence for one public control and two identity-sealed evaluated lanes. It has no public preference result. - The two tracks are reported separately and never collapsed into one leaderboard. ## Standing and uncertainty -The canonical Arena score is Bradley-Terry estimated win probability against Sol, normalized so Sol is exactly 50.0. Uncertainty uses 2,000 deterministic source-family cluster-bootstrap replicates at 95% confidence. The public preview demonstrates that presentation with synthetic numbers only. +The canonical Arena estimator is Bradley-Terry logistic maximum likelihood over governed resolved battles. Uncertainty uses 2,000 deterministic source-family cluster-bootstrap replicates at 95% confidence. A future governed release must declare its score calibration and reference baseline; this preview publishes no score, interval, or model standing. -An ordinal standing is forbidden when the frozen battle, seven-source-family, human-judge, complete-roster, connectivity, bootstrap, contamination, or position-bias gates are not met. `tied` means a pairwise 95% interval contains zero; `insufficient_evidence` means one or more gates failed. Synthetic preview rows remain `publishable=false`, `rank_count=0`, and non-rankable regardless of their illustrative state. +An ordinal standing is forbidden when the frozen battle, seven-source-family, human-judge, complete-roster, connectivity, bootstrap, contamination, or position-bias gates are not met. `tied` means a pairwise 95% interval contains zero; `insufficient_evidence` means one or more gates failed. The current release has no rows and remains `publishable=false`, `rank_count=0`, `winner=null`, and non-rankable. Deterministic task success is computed independently from preference battles. Failure rate retains explicit terminal failures in the denominator. Latency, token totals, and estimated cost use frozen run records; they never disappear because a run failed. ## Anonymous battle replay -The replay visually hides model identity until the visitor records `A is better`, `B is better`, `Tie`, or `Both bad`. A/B order can be swapped before judgment. The judgment is stored only in React state for the current browser view: there is no API request, persistence, identity, anti-abuse control, or official ranking effect. Synthetic identities are present in the downloadable fixture, so preview reveal is not a secrecy boundary; a governed backend must withhold real identity payloads until judgment. +The replay is a generic synthetic interaction demo. It records `A is better`, `B is better`, `Tie`, `Both bad`, or `Abstain`; A/B order can be swapped before judgment. The choice is stored only in React state for the current browser view: there is no API request, persistence, identity, anti-abuse control, or official ranking effect. + +The real blind packet, candidate media, per-cell identifiers, output hashes, and identity mapping remain private until the founder votes. They are not represented by `battles.preview.jsonl`, and a static public artifact must never be treated as an identity-secrecy boundary. Any future governed vote backend requires a reviewed sampling policy, replay identity, duplicate/abuse controls, immutable battle and output hashes, model reveal rules, audit export, deletion/privacy handling, and a versioned policy that says exactly which votes enter an official release. @@ -37,8 +39,10 @@ Public-eval and private-test source families must remain disjoint. Hidden-test f - System-eval contract SHA-256: `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624` - FineVideo selection release SHA-256: `d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926` - FineVideo selection-manifest SHA-256: `39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4` +- Seven-family staging-manifest SHA-256: `0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8` +- Seven-family case-pack SHA-256: `74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203` -The separate `FINEVIDEO_RELEASED` marker now exists and the official release check passes for 24 cells with the contract and release hashes above. That gate authorizes only the frozen source cells. The rankable runner/runtime has not yet produced real model records, so this public surface still publishes no measured score, output, or rank. +The separate `FINEVIDEO_RELEASED` marker exists and the official release check passes for 24 cells with the contract and release hashes above. Seven eligible cells from seven distinct families completed the deterministic protocol with no favorable selection. The public aggregate still publishes no candidate output, preference score, identity mapping, or rank. ## Contamination and privacy boundaries @@ -48,10 +52,9 @@ Public exports may contain aggregate metrics, public-safe artifact hashes, requi ## Known limitations -- All preview scores, intervals, costs, and outputs are synthetic. -- Phase 1 has one complete model output plus a mechanical control, but no complete seven-family required roster and therefore no blind battle or preference score. -- The dated Phase 1 pilot evidence is deterministic task-success evidence with zero blind battles, zero ranks, and zero human preference claims. +- No preview score, interval, cost, model rank, or winner is published. +- Seven-family task-success counts are deterministic system evidence, not editorial preference. +- Evaluated lane identities remain sealed pending the founder's blind vote. - The public surface does not implement an official vote backend. - The frozen system-eval contract currently uses one repetition per real cell; reliability claims need a reviewed repeated-run extension. -- Kimi K3 uses Composer's expressible `high` effort because the runtime cannot express the provider's published `max` effort. - Editorial preference and long-horizon recovery remain exploratory, non-rankable capability views. diff --git a/public/bench/composer-arena/v0/release.preview.json b/public/bench/composer-arena/v0/release.preview.json index 6a640ed..90115b7 100644 --- a/public/bench/composer-arena/v0/release.preview.json +++ b/public/bench/composer-arena/v0/release.preview.json @@ -59,143 +59,15 @@ "id": "controlled-agent", "canonical_track_id": "controlled-semantic-text-evidence-v0", "title": "Controlled agent", - "description": "Synthetic interface examples only. The real controlled-semantic track is blocked until one genuine, hash-identical transcript/OCR/semantic-event artifact is frozen for all four systems.", - "rows": [ - { - "standing": "tied", - "display_rank": "—", - "model": "GPT-5.6 Sol", - "provider": "AgenticFlow", - "version": "sol-medium-v1", - "evaluation_date": "2026-07-19", - "arena_score": 50, - "arena_score_ci95": [47.7, 52.3], - "battle_count": 48, - "deterministic_task_success": 0.833, - "failure_rate": 0.063, - "latency_ms_p50": 42100, - "tokens": { "input": 18400, "output": 6100, "cache_read": 9200, "cache_write": 0 }, - "estimated_cost_usd_per_run": 0.72, - "per_task_slices": [ - { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.875, "battle_count": 16 }, - { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.813, "battle_count": 16 }, - { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.813, "battle_count": 16 } - ] - }, - { - "standing": "tied", - "display_rank": "—", - "model": "Grok 4.5", - "provider": "AgenticFlow", - "version": "grok-4.5-medium-v1", - "evaluation_date": "2026-07-19", - "arena_score": 52.4, - "arena_score_ci95": [49.7, 55.1], - "battle_count": 48, - "deterministic_task_success": 0.813, - "failure_rate": 0.083, - "latency_ms_p50": 39700, - "tokens": { "input": 17600, "output": 6800, "cache_read": 7600, "cache_write": 0 }, - "estimated_cost_usd_per_run": 0.66, - "per_task_slices": [ - { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.813, "battle_count": 16 }, - { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.813, "battle_count": 16 }, - { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.813, "battle_count": 16 } - ] - }, - { - "standing": "insufficient_evidence", - "display_rank": "—", - "model": "Kimi K3", - "provider": "AgenticFlow", - "version": "kimi-k3-high-v1", - "evaluation_date": "2026-07-19", - "arena_score": 49.1, - "arena_score_ci95": [41.2, 57], - "battle_count": 8, - "deterministic_task_success": 0.75, - "failure_rate": 0.125, - "latency_ms_p50": 51400, - "tokens": { "input": 20100, "output": 7200, "cache_read": 8100, "cache_write": 0 }, - "estimated_cost_usd_per_run": 0.81, - "per_task_slices": [ - { "task_id": "archive-grounding", "label": "Archive grounding", "success_rate": 0.75, "battle_count": 4 }, - { "task_id": "temporal-ordering", "label": "Temporal ordering", "success_rate": 0.75, "battle_count": 2 }, - { "task_id": "executable-plan", "label": "Executable plan", "success_rate": 0.75, "battle_count": 2 } - ] - } - ] + "description": "No public standing exists. The released evidence pack contains no transcript, OCR, or semantic events, so this track remains blocked before model execution.", + "rows": [] }, { "id": "end-to-end-system", "canonical_track_id": "end-to-end-visual-orchestration-v0", "title": "End-to-end system", - "description": "Synthetic interface examples for the separate visual-orchestration track. Real results require complete Sol, Grok 4.5, and Kimi K3 outputs on every frozen holdout stratum.", - "rows": [ - { - "standing": "tied", - "display_rank": "—", - "model": "GPT-5.6 Sol + Composer", - "provider": "AgenticFlow / Pixel ML", - "version": "sol-medium-v1 / composer-v0", - "evaluation_date": "2026-07-19", - "arena_score": 50, - "arena_score_ci95": [46.9, 53.1], - "battle_count": 36, - "deterministic_task_success": 0.778, - "failure_rate": 0.111, - "latency_ms_p50": 118400, - "tokens": { "input": 26300, "output": 8700, "cache_read": 11200, "cache_write": 0 }, - "estimated_cost_usd_per_run": 1.18, - "per_task_slices": [ - { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.833, "battle_count": 12 }, - { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 0.833, "battle_count": 12 }, - { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.667, "battle_count": 12 } - ] - }, - { - "standing": "tied", - "display_rank": "—", - "model": "Grok 4.5 + Composer", - "provider": "AgenticFlow / Pixel ML", - "version": "grok-4.5-medium-v1 / composer-v0", - "evaluation_date": "2026-07-19", - "arena_score": 51.6, - "arena_score_ci95": [48.4, 54.8], - "battle_count": 36, - "deterministic_task_success": 0.75, - "failure_rate": 0.139, - "latency_ms_p50": 109600, - "tokens": { "input": 24900, "output": 9100, "cache_read": 9800, "cache_write": 0 }, - "estimated_cost_usd_per_run": 1.09, - "per_task_slices": [ - { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.75, "battle_count": 12 }, - { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 0.833, "battle_count": 12 }, - { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.667, "battle_count": 12 } - ] - }, - { - "standing": "insufficient_evidence", - "display_rank": "—", - "model": "Kimi K3 + Composer", - "provider": "AgenticFlow / Pixel ML", - "version": "kimi-k3-high-v1 / composer-v0", - "evaluation_date": "2026-07-19", - "arena_score": 48.8, - "arena_score_ci95": [39.6, 58.1], - "battle_count": 6, - "deterministic_task_success": 0.667, - "failure_rate": 0.167, - "latency_ms_p50": 126800, - "tokens": { "input": 23100, "output": 7900, "cache_read": 7400, "cache_write": 0 }, - "estimated_cost_usd_per_run": 0.94, - "per_task_slices": [ - { "task_id": "timeline-validity", "label": "Timeline validity", "success_rate": 0.667, "battle_count": 2 }, - { "task_id": "source-provenance", "label": "Source provenance", "success_rate": 1, "battle_count": 2 }, - { "task_id": "reviewable-delivery", "label": "Reviewable delivery", "success_rate": 0.333, "battle_count": 2 } - ] - } - ] + "description": "Seven-family deterministic task-success evidence exists, but no governed blind preference result or public standing exists.", + "rows": [] } ], "exploratory_capabilities": [ @@ -203,17 +75,17 @@ "id": "direct-perception", "title": "Direct visual perception", "status": "non_rankable", - "summary": "Can the route inspect released frames directly rather than reason only over frozen derived evidence?", - "evidence_scope": "Capability declaration and preflight only; no synthetic score is converted into a rank.", - "limitation": "GLM 5.2 is marked not-applicable for direct perception in the frozen contract, which is not treated as a failure." + "summary": "Can a route inspect released frames directly rather than reason only over frozen derived evidence?", + "evidence_scope": "Capability declaration and preflight only; no capability statement is converted into rank.", + "limitation": "Unavailable direct perception is recorded as not applicable rather than an editing-quality failure." }, { "id": "editorial-preference", "title": "Editorial preference", "status": "non_rankable", "summary": "Blind human review of coherence, opening strength, cut naturalness, and caption readability.", - "evidence_scope": "Two reviewers plus adjudication are required for any real public editorial claim.", - "limitation": "Human preference remains separate from deterministic task success and the synthetic Arena score." + "evidence_scope": "A private, identity-sealed packet and a recorded human judgment are required before any reveal.", + "limitation": "Deterministic task success is not editorial preference and never creates a winner." }, { "id": "long-horizon-recovery", @@ -221,15 +93,15 @@ "status": "non_rankable", "summary": "Recovery from tool failures, stale state, and interrupted editing turns.", "evidence_scope": "Exploratory fault-injection notes only.", - "limitation": "The v0 public contract does not yet freeze a comparable failure-injection distribution." + "limitation": "The v0 public contract does not freeze a comparable failure-injection distribution." } ], "known_limitations": [ - "All scores, confidence intervals, battle counts, costs, and outputs in this preview are synthetic interface fixtures and must not be interpreted as measured model performance.", - "Phase 1 produced one complete model output plus a mechanical control, but no complete seven-family required roster or blind human comparison; every preview-table row remains synthetic and non-rankable.", - "One repetition per real cell is currently frozen in the system-eval contract; reliability claims require a reviewed repeated-run extension.", - "Kimi K3 is evaluated at Composer's expressible high effort because the runtime cannot express the provider's published max effort.", - "Local replay judgments are browser-only and have no governed identity, anti-abuse, sampling, or persistence backend.", - "Preview identity reveal is a visual interaction, not a secrecy boundary: synthetic candidate identities ship in the downloadable fixture. A governed backend must withhold real identities until judgment." + "No public Arena score, confidence interval, battle count, model rank, cost comparison, or winner exists.", + "Seven-family task-success evidence is deterministic system evidence, not a preference result.", + "The controlled-agent track remains blocked because the released evidence pack has no transcript, OCR, or semantic events.", + "The local replay is a synthetic interaction demo with generic candidates and has no governed identity, sampling, persistence, or ranking effect.", + "The real blind packet, candidate outputs, and model identity mapping remain private until the founder records a vote.", + "One repetition per real cell is frozen; reliability claims require a reviewed repeated-run extension." ] } diff --git a/public/bench/composer-arena/v0/release.schema.json b/public/bench/composer-arena/v0/release.schema.json index ec35d7c..0d30e49 100644 --- a/public/bench/composer-arena/v0/release.schema.json +++ b/public/bench/composer-arena/v0/release.schema.json @@ -144,7 +144,7 @@ "description": { "type": "string", "minLength": 1 }, "rows": { "type": "array", - "minItems": 1, + "minItems": 0, "items": { "$ref": "#/$defs/row" } } } diff --git a/scripts/check-composer-arena.mjs b/scripts/check-composer-arena.mjs index 4ceecb0..4e6c631 100644 --- a/scripts/check-composer-arena.mjs +++ b/scripts/check-composer-arena.mjs @@ -7,21 +7,26 @@ const root = resolve(import.meta.dirname, ".."); const routeRoot = resolve(root, "src/app/bench/composer-arena"); const artifactRoot = resolve(root, "public/bench/composer-arena/v0"); const evidenceArtifactRoot = resolve(root, "public/bench/composer-arena/evidence/pilot/2026-07-19"); + const release = JSON.parse(readFileSync(resolve(artifactRoot, "release.preview.json"), "utf8")); -const schema = JSON.parse(readFileSync(resolve(artifactRoot, "release.schema.json"), "utf8")); +const releaseSchema = JSON.parse(readFileSync(resolve(artifactRoot, "release.schema.json"), "utf8")); +const battleSchema = JSON.parse(readFileSync(resolve(artifactRoot, "battle.schema.json"), "utf8")); const checksums = JSON.parse(readFileSync(resolve(artifactRoot, "checksums.json"), "utf8")); const battles = readFileSync(resolve(artifactRoot, "battles.preview.jsonl"), "utf8") .split("\n") .map((line) => line.trim()) .filter(Boolean) .map((line) => JSON.parse(line)); + const page = readFileSync(resolve(routeRoot, "page.tsx"), "utf8"); const client = readFileSync(resolve(routeRoot, "arena-client.tsx"), "utf8"); const data = readFileSync(resolve(routeRoot, "data.ts"), "utf8"); +const benchPage = readFileSync(resolve(root, "src/app/bench/page.tsx"), "utf8"); const methodology = readFileSync(resolve(artifactRoot, "methodology.md"), "utf8"); const handoff = readFileSync(resolve(artifactRoot, "handoff.md"), "utf8"); -const evidence = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "phase-1-results.json"), "utf8")); -const evidenceSchema = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "phase-1-results.schema.json"), "utf8")); + +const evidence = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "seven-family-results.json"), "utf8")); +const evidenceSchema = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "seven-family-results.schema.json"), "utf8")); const evidenceChecksums = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "checksums.json"), "utf8")); const evidencePage = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/page.tsx"), "utf8"); const evidenceData = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/data.ts"), "utf8"); @@ -40,6 +45,14 @@ function sha256(base, filename) { return createHash("sha256").update(readFileSync(resolve(base, filename))).digest("hex"); } +function requireChecksumSet(base, checksumDocument, label) { + for (const [filename, expected] of Object.entries(checksumDocument.files)) { + requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid ${label} checksum for ${filename}`); + requireCondition(existsSync(resolve(base, filename)), `missing checksummed ${label} artifact: ${filename}`); + requireCondition(sha256(base, filename) === expected, `${label} checksum mismatch for ${filename}`); + } +} + const requiredArtifacts = [ "release.schema.json", "battle.schema.json", @@ -55,37 +68,26 @@ for (const artifact of requiredArtifacts) { requireCondition(page.includes(artifact), `Composer Arena page does not link ${artifact}`); } requireCondition(page.includes('const ARTIFACT_ROOT = "/bench/composer-arena/v0"'), "Composer Arena artifact root drifted"); +requireChecksumSet(artifactRoot, checksums, "Composer Arena"); +requireChecksumSet(evidenceArtifactRoot, evidenceChecksums, "seven-family evidence"); -for (const [filename, expected] of Object.entries(checksums.files)) { - requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid checksum for ${filename}`); - requireCondition(sha256(artifactRoot, filename) === expected, `checksum mismatch for ${filename}`); -} - -for (const [filename, expected] of Object.entries(evidenceChecksums.files)) { - requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid pilot checksum for ${filename}`); - requireCondition(sha256(evidenceArtifactRoot, filename) === expected, `pilot checksum mismatch for ${filename}`); -} - -requireCondition(schema.$id === "https://agentic.video/bench/composer-arena/v0/release.schema.json", "schema ID drifted"); -requireCondition(schema.additionalProperties === false, "release schema must fail closed on unknown top-level fields"); -validateSchema(schema, release, "preview release"); -const battleSchema = JSON.parse(readFileSync(resolve(artifactRoot, "battle.schema.json"), "utf8")); +requireCondition(releaseSchema.$id === "https://agentic.video/bench/composer-arena/v0/release.schema.json", "release schema ID drifted"); +requireCondition(releaseSchema.additionalProperties === false, "release schema must fail closed on unknown top-level fields"); +validateSchema(releaseSchema, release, "preview release"); for (const battle of battles) validateSchema(battleSchema, battle, `battle ${battle.battle_id}`); + requireCondition(release.release_mode === "synthetic_non_rankable_preview", "fixtures must remain a synthetic preview"); requireCondition(release.data_class === "synthetic", "preview data class must remain synthetic"); -requireCondition(release.official_ranking_enabled === false, "official ranking must remain disabled for fixtures"); +requireCondition(release.official_ranking_enabled === false, "official ranking must remain disabled"); requireCondition(release.official_vote_backend === "not_configured", "preview must not imply a governed vote backend"); requireCondition(release.display_label === "non-rankable preview", "preview label drifted"); +requireCondition(release.publication.publishable === false, "preview must not be publishable"); +requireCondition(release.publication.state === "insufficient_evidence", "preview publication state must fail closed"); +requireCondition(release.publication.winner === null && release.publication.rank_count === 0, "preview must have no winner or rank"); +requireCondition(release.vote_provenance.real_human_votes === 0, "preview must have zero real human votes"); requireCondition(release.dataset.repo_id === "HuggingFaceFV/finevideo", "FineVideo repo drifted"); requireCondition(release.dataset.revision === "84c74091e1c6ee7a5dffabfafb5c9033e4718883", "FineVideo revision drifted"); requireCondition(release.dataset.license_claim === "CC-BY", "FineVideo licence claim drifted"); -requireCondition(release.arena_binding.source_head_sha === "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", "Arena source head drifted"); -requireCondition(release.arena_binding.arena_contract_sha256 === "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", "Arena contract hash drifted"); -requireCondition(release.arena_binding.parent_contract_sha256 === "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", "parent contract hash drifted"); -requireCondition(release.publication.publishable === false, "synthetic preview must not be publishable"); -requireCondition(release.publication.state === "insufficient_evidence", "synthetic preview publication state must fail closed"); -requireCondition(release.publication.winner === null && release.publication.rank_count === 0, "synthetic preview must have no winner or rank"); -requireCondition(release.vote_provenance.collection_mode === "synthetic_fixture" && release.vote_provenance.real_human_votes === 0, "synthetic preview vote provenance drifted"); requireCondition(release.dataset.finevideo_release_sha256 === "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", "FineVideo release hash drifted"); requireCondition(release.dataset.split_policy.source_family_disjoint === true, "source families must remain disjoint"); requireCondition(release.dataset.split_policy.hidden_test_published === false, "hidden test must remain unpublished"); @@ -93,99 +95,171 @@ requireCondition(release.dataset.split_policy.hidden_test_published === false, " const trackIds = release.tracks.map((track) => track.id).sort(); requireCondition( JSON.stringify(trackIds) === JSON.stringify(["controlled-agent", "end-to-end-system"]), - "release must contain exactly the two separate Arena tracks", + "release must contain exactly two separate Arena tracks", ); const canonicalTrackIds = release.tracks.map((track) => track.canonical_track_id).sort(); requireCondition( JSON.stringify(canonicalTrackIds) === JSON.stringify(["controlled-semantic-text-evidence-v0", "end-to-end-visual-orchestration-v0"]), "canonical Arena track IDs drifted", ); - for (const track of release.tracks) { - requireCondition(track.rows.length > 0, `${track.id} has no rows`); - requireCondition(track.rows.some((row) => row.standing === "tied"), `${track.id} must show a visible tied example`); - requireCondition( - track.rows.some((row) => row.standing === "insufficient_evidence"), - `${track.id} must show insufficient evidence`, - ); - - for (const row of track.rows) { - requireCondition(row.model && row.provider && row.version && row.evaluation_date, `${track.id} row identity is incomplete`); - requireCondition(Number.isFinite(row.arena_score) && row.arena_score >= 0 && row.arena_score <= 100, `${track.id} row has an invalid preview score`); - requireCondition(Array.isArray(row.arena_score_ci95) && row.arena_score_ci95.length === 2, `${track.id} row CI is invalid`); - requireCondition(Number.isInteger(row.battle_count) && row.battle_count >= 0, `${track.id} battle count is invalid`); - requireCondition(row.deterministic_task_success >= 0 && row.deterministic_task_success <= 1, `${track.id} task success is invalid`); - requireCondition(row.failure_rate >= 0 && row.failure_rate <= 1, `${track.id} failure rate is invalid`); - requireCondition(row.latency_ms_p50 >= 0, `${track.id} latency is invalid`); - requireCondition(Object.values(row.tokens).every((value) => Number.isInteger(value) && value >= 0), `${track.id} token usage is invalid`); - requireCondition(row.estimated_cost_usd_per_run >= 0, `${track.id} cost is invalid`); - requireCondition(row.per_task_slices.length > 0, `${track.id} per-task slices are missing`); - } + requireCondition(track.rows.length === 0, `${track.id} must publish zero standing rows`); } - requireCondition( release.exploratory_capabilities.every((capability) => capability.status === "non_rankable"), "exploratory capabilities must remain non-rankable", ); -requireCondition(battles.length >= 2, "preview needs controlled-agent and end-to-end battle replays"); -requireCondition(new Set(battles.map((battle) => battle.track)).size === 2, "battle fixtures must cover both tracks"); + +requireCondition(battles.length === 2, "the generic demo must contain exactly two replay fixtures"); +requireCondition(new Set(battles.map((battle) => battle.track)).size === 2, "demo fixtures must cover both tracks"); for (const battle of battles) { requireCondition(battle.source.rankable === false, `${battle.battle_id} must remain non-rankable`); requireCondition(battle.left.evidence.length > 0 && battle.right.evidence.length > 0, `${battle.battle_id} lacks evidence links`); + for (const candidate of [battle.left, battle.right]) { + requireCondition(candidate.model.startsWith("Synthetic demo "), `${battle.battle_id} leaks a non-generic candidate identity`); + requireCondition(candidate.provider === "Not applicable", `${battle.battle_id} leaks a provider identity`); + requireCondition(candidate.version.startsWith("fixture-"), `${battle.battle_id} leaks a route version`); + } +} + +const serializedPreview = JSON.stringify({ release, battles }).toLowerCase(); +for (const forbidden of [ + "gpt-5.6", + "grok 4.5", + "kimi k3", + "sol-medium-v1", + "grok-4.5-medium-v1", + "kimi-k3-high-v1", +]) { + requireCondition(!serializedPreview.includes(forbidden), `public preview contains a real-model fixture identity: ${forbidden}`); } -requireCondition(client.includes("Swap A/B order"), "battle replay must support swapped order"); -requireCondition(client.includes("Both bad"), "battle replay must support both-bad judgments"); -requireCondition(client.includes("Tie"), "battle replay must support tie judgments"); -requireCondition(client.includes("revealed={judgment !== null}"), "model identity must reveal only after judgment"); +for (const control of ["Swap A/B order", "A is better", "B is better", "Tie", "Both bad", "Abstain"]) { + requireCondition(client.includes(control), `local demo is missing the ${control} control`); +} +requireCondition(client.includes("revealed={judgment !== null}"), "demo identity must reveal only after judgment"); requireCondition(client.includes("No request is sent"), "local vote exclusion must be visible"); requireCondition(data.includes("release.preview.json") && data.includes("battles.preview.jsonl"), "route must be artifact-driven"); + +requireCondition(methodology.includes("seven-family task-success"), "methodology is missing seven-family evidence"); requireCondition(methodology.includes("source families must remain disjoint"), "methodology is missing the source-family split"); requireCondition(methodology.includes("CC-BY"), "methodology is missing FineVideo attribution scope"); -requireCondition(methodology.includes("official release check passes for 24 cells"), "methodology must bind the green source release"); -requireCondition(methodology.includes("runner/runtime"), "methodology must disclose the missing real runner records"); -requireCondition(handoff.includes("without changing React code"), "handoff must preserve schema-driven replacement"); +requireCondition(methodology.includes("Abstain"), "methodology is missing the abstain path"); +requireCondition(methodology.includes("identity mapping remain private"), "methodology is missing the sealed-identity boundary"); requireCondition(handoff.includes("battle.schema.json"), "handoff must bind JSONL rows to the battle schema"); requireCondition(handoff.includes("governed vote backend"), "handoff is missing official vote governance"); -requireCondition(handoff.includes("fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5"), "handoff is not bound to the Arena contract"); -requireCondition(handoff.includes("controlled-semantic-text-evidence-v0") && handoff.includes("end-to-end-visual-orchestration-v0"), "handoff is missing canonical track separation"); requireCondition(handoff.includes("SEA Broadcast ASR remains a separate benchmark suite"), "handoff must preserve the SEA-ASR boundary"); -requireCondition(!page.includes("customer data") || page.includes("never"), "page must not normalize customer data use"); - -requireCondition(evidenceSchema.additionalProperties === false, "pilot evidence schema must fail closed"); -validateSchema(evidenceSchema, evidence, "Phase 1 pilot evidence"); -requireCondition(evidence.arena_binding.source_head_sha === "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", "pilot source head drifted"); -requireCondition(evidence.arena_binding.arena_contract_sha256 === "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", "pilot Arena hash drifted"); -requireCondition(evidence.publication.publishable === false, "pilot evidence must not be publishable"); -requireCondition(evidence.publication.state === "insufficient_evidence", "pilot evidence must remain insufficient"); -requireCondition(evidence.publication.leaderboard === false, "pilot evidence must not be a leaderboard"); -requireCondition(evidence.publication.winner === null, "pilot evidence must not imply a winner"); -requireCondition(evidence.publication.rank_count === 0, "pilot evidence must publish zero ranks"); -requireCondition(evidence.publication.human_preference_claims === false, "pilot evidence must publish zero human preference claims"); +requireCondition(handoff.includes("abstain"), "handoff is missing the abstain path"); +requireCondition(handoff.includes("outside `public/`"), "handoff is missing the private blind-packet boundary"); +requireCondition(benchPage.includes("Separate benchmark suite"), "SEA-ASR page must label Composer as a separate suite"); +requireCondition(benchPage.includes("does not change SEA Broadcast ASR"), "SEA-ASR page must state that Composer cannot change its result"); + +requireCondition(evidenceSchema.additionalProperties === false, "seven-family evidence schema must fail closed"); +validateSchema(evidenceSchema, evidence, "seven-family evidence"); +requireCondition(evidence.evidence_id === "composer-arena-seven-family-task-success-2026-07-19", "evidence ID drifted"); +requireCondition(evidence.publication.publishable === false, "seven-family evidence must not be publishable"); +requireCondition(evidence.publication.state === "insufficient_evidence", "seven-family evidence must remain insufficient"); +requireCondition(evidence.publication.leaderboard === false, "seven-family evidence must not be a leaderboard"); +requireCondition(evidence.publication.winner === null && evidence.publication.rank_count === 0, "seven-family evidence must have zero ranks and no winner"); +requireCondition(evidence.publication.human_preference_claims === false, "seven-family evidence must make no human preference claim"); requireCondition( evidence.vote_provenance.blind_battles === 0 && evidence.vote_provenance.real_human_votes === 0 && evidence.vote_provenance.resolved_battle_groups === 0, - "pilot evidence must publish zero battles and votes", + "seven-family evidence must publish zero battles and votes", +); +requireCondition( + evidence.cohort.eligible_cells === 7 && + evidence.cohort.distinct_source_families === 7 && + evidence.cohort.favorable_selection === false && + evidence.cohort.source_family_disjoint === true, + "seven-family cohort gate drifted", ); -const pilotResults = Object.fromEntries(evidence.task_success.results.map((result) => [result.baseline_id, result])); -requireCondition(pilotResults["chronological-mechanical-v1"].terminal_status === "passed" && pilotResults["chronological-mechanical-v1"].output?.complete === true, "mechanical control evidence drifted"); -requireCondition(pilotResults["grok-4.5-medium-v1"].terminal_status === "passed" && pilotResults["grok-4.5-medium-v1"].output?.complete === true, "Grok Phase 1 evidence drifted"); -requireCondition(pilotResults["sol-medium-v1"].failure_code === "tool_denied" && pilotResults["sol-medium-v1"].task_outcome === "policy_task_failure", "Sol policy/task failure drifted"); -requireCondition(pilotResults["kimi-k3-high-v1"].terminal_status === "excluded" && pilotResults["kimi-k3-high-v1"].quality_loss === false, "Kimi compatibility exclusion drifted"); -requireCondition(pilotResults["glm-5.2-high-v1"].terminal_status === "not_applicable" && pilotResults["glm-5.2-high-v1"].quality_loss === false, "GLM direct-vision N/A drifted"); -requireCondition(evidence.diagnostic_limitations.hyperframes_version === "0.7.64", "HyperFrames diagnostic version drifted"); -requireCondition(evidence.diagnostic_limitations.mechanical_control_diagnostic_count === 8 && evidence.diagnostic_limitations.grok_4_5_diagnostic_count === 6, "HyperFrames diagnostic counts drifted"); - -const serializedEvidence = JSON.stringify(evidence); -for (const forbidden of ["/Users/", "/tmp/", "http://", "https://", "s3://", "signed_url", "credential", "password", "secret", "hidden_test"]) { - requireCondition(!serializedEvidence.toLowerCase().includes(forbidden.toLowerCase()), `pilot evidence contains forbidden public material: ${forbidden}`); +const expectedHashes = { + staging_manifest_sha256: "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8", + contract_sha256: "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", + release_sha256: "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + case_pack_sha256: "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203", +}; +for (const [key, expected] of Object.entries(expectedHashes)) { + requireCondition(evidence.frozen_evidence[key] === expected, `seven-family ${key} drifted`); +} + +const lanes = Object.fromEntries(evidence.task_success.lanes.map((lane) => [lane.lane_id, lane])); +const expectedLanes = { + "mechanical-control": { + classification: "control", + identity_state: "public_control", + structural_passes: 6, + structural_failures: 1, + complete_outputs_accepted: 5, + complete_output_failures: 2, + model_calls: 0, + source_summary_sha256: "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57", + }, + "evaluated-lane-a": { + classification: "evaluated_system", + identity_state: "sealed_pending_founder_vote", + structural_passes: 2, + structural_failures: 5, + complete_outputs_accepted: 2, + complete_output_failures: 5, + model_calls: 7, + source_summary_sha256: "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a", + }, + "evaluated-lane-b": { + classification: "evaluated_system", + identity_state: "sealed_pending_founder_vote", + structural_passes: 4, + structural_failures: 3, + complete_outputs_accepted: 4, + complete_output_failures: 3, + model_calls: 7, + source_summary_sha256: "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456", + }, +}; +requireCondition(Object.keys(lanes).length === 3, "seven-family evidence must contain exactly three lanes"); +for (const [laneId, expected] of Object.entries(expectedLanes)) { + const lane = lanes[laneId]; + requireCondition(lane?.attempted_cells === 7, `${laneId} must contain seven attempted cells`); + for (const [key, value] of Object.entries(expected)) { + requireCondition(lane[key] === value, `${laneId} ${key} drifted`); + } +} + +for (const flag of ["packet_public", "candidate_outputs_public", "identity_mapping_public", "founder_vote_recorded"]) { + requireCondition(evidence.blind_review[flag] === false, `blind-review flag ${flag} must remain false`); +} + +const serializedEvidence = JSON.stringify(evidence).toLowerCase(); +for (const forbidden of [ + "/users/", + "/tmp/", + "http://", + "https://", + "s3://", + "signed_url", + "credential", + "password", + "secret", + "hidden_test", + "finevideo-f567", + "run_id", + "output_sha256", + "candidate_id", + "model_id", + "provider", +]) { + requireCondition(!serializedEvidence.includes(forbidden), `seven-family evidence contains forbidden public material: ${forbidden}`); +} + +requireCondition(evidencePage.includes("not a leaderboard"), "evidence route must visibly reject leaderboard framing"); +requireCondition(evidencePage.includes("Winner: none"), "evidence route must visibly reject a winner claim"); +for (const label of ["Source families", "Blind battles", "Published ranks", "Human votes"]) { + requireCondition(evidencePage.includes(label), `evidence route is missing the ${label} count`); } -requireCondition(evidencePage.includes("not a leaderboard"), "pilot route must visibly reject leaderboard framing"); -requireCondition(evidencePage.includes("Winner: none"), "pilot route must visibly reject a winner claim"); -requireCondition(evidencePage.includes("Blind battles") && evidencePage.includes("Published ranks") && evidencePage.includes("Human preference claims"), "pilot route must show all zero claim counts"); -requireCondition(evidencePage.includes("HyperFrames compatibility finding"), "pilot route must disclose the HyperFrames limitation"); -requireCondition(evidenceData.includes("phase-1-results.json"), "pilot route must be artifact-driven"); +requireCondition(evidencePage.includes("no candidate media"), "evidence route must disclose the private packet boundary"); +requireCondition(evidenceData.includes("seven-family-results.json"), "evidence route must be artifact-driven"); console.log("Composer Arena route contract check passed"); diff --git a/src/app/bench/composer-arena/arena-client.tsx b/src/app/bench/composer-arena/arena-client.tsx index bf85a96..b197520 100644 --- a/src/app/bench/composer-arena/arena-client.tsx +++ b/src/app/bench/composer-arena/arena-client.tsx @@ -9,7 +9,7 @@ import type { BattleCandidate, } from "./types"; -type Judgment = "left" | "right" | "tie" | "both_bad"; +type Judgment = "left" | "right" | "tie" | "both_bad" | "abstain"; const percent = new Intl.NumberFormat("en-US", { style: "percent", @@ -47,6 +47,21 @@ function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: Aren if (!track) return null; + if (track.rows.length === 0) { + return ( +
+
+

{track.title}

+

{track.description}

+ {track.canonical_track_id} +
+
+ No public standings are available. Task-success evidence stays separate from preference ranking. +
+
+ ); + } + return (
@@ -63,7 +78,7 @@ function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: Aren {[ "Standing", "Model / route", - "Preview score vs Sol (95% CI)", + "Arena score (95% CI)", "Battles", "Task success", "Failure rate", @@ -252,12 +267,13 @@ function BattleReplay({ battles }: { battles: ArenaBattle[] }) {

This selection stays in this page only. No request is sent and official standings never change.

-
+
{([ ["left", "A is better"], ["tie", "Tie"], ["right", "B is better"], ["both_bad", "Both bad"], + ["abstain", "Abstain"], ] as const).map(([value, label]) => (
@@ -51,21 +44,22 @@ export default async function PhaseOneEvidencePage() { Composer Archive-to-Output Arena v0

- Phase 1 evidence pilot + Seven-family task-success evidence

- A sanitized, dated record of deterministic task outcomes and complete-output acceptance on one audited public-safe cell. It cannot establish model preference or an Arena winner. + A sanitized aggregate over seven eligible cells from seven distinct source families. It reports deterministic system outcomes only; it cannot establish editorial preference or an Arena winner.

- Winner: none. Publication state: insufficient evidence. No blind comparison exists because the required visual roster is incomplete. + Winner: none. Publication state: insufficient evidence. Evaluated identities and the real blind packet remain private until the founder records a vote.
-
+
{[ + ["Source families", evidence.cohort.distinct_source_families], ["Blind battles", evidence.vote_provenance.blind_battles], ["Published ranks", evidence.publication.rank_count], - ["Human preference claims", evidence.publication.human_preference_claims ? 1 : 0], + ["Human votes", evidence.vote_provenance.real_human_votes], ].map(([term, value]) => (
{term}
@@ -76,126 +70,81 @@ export default async function PhaseOneEvidencePage() {
-
+

Deterministic task success

-

Five outcomes; no preference ordering

+

Three lanes; no preference ordering

{evidence.task_success.separation_statement}

-
- {evidence.task_success.results.map((result) => ( -
-
-
-

{result.display_name}

- {result.baseline_id} -
- - {result.terminal_status.replace("_", " ")} - -
-

{result.reason}

-
-
-
Scientific treatment
-
{result.task_outcome.replaceAll("_", " ")}
-
-
-
Preference eligible
-
No
-
- {result.run_id && ( -
-
Run ID
-
{result.run_id}
-
- )} - {result.failure_code && ( -
-
Failure code
-
{result.failure_code}
-
- )} - {result.latency_ms !== null && ( -
-
Elapsed
-
{(result.latency_ms / 1000).toFixed(3)}s
-
- )} -
-
- ))} -
-
-
- -
-
-

Complete-output acceptance

-

Two outputs passed every acceptance gate

- + - {['Candidate', 'Output', 'Decode', 'Opening black', 'A/V drift', 'Output SHA-256', 'Acceptance SHA-256'].map((heading) => ( + {[ + "Lane", + "Identity", + "Attempted", + "Structural", + "Accepted outputs", + "Model calls", + "Source summary SHA-256", + ].map((heading) => ( ))} - {completedOutputs.map((result) => { - const output = result.output!; - return ( - - - - - - - - - - ); - })} + {evidence.task_success.lanes.map((lane) => ( + + + + + + + + + + ))}
Accepted Phase 1 outputsSeven-family deterministic task-success lanes
{heading}
{result.display_name}{(output.duration_ms / 1000).toFixed(3)}s
{output.bytes.toLocaleString()} bytes
{output.video_codec}/{output.audio_codec} · {output.format}		PASS{output.opening_black_ms}ms{output.av_start_drift_ms}ms start
{output.av_end_drift_ms}ms end		{output.output_sha256}{output.render_acceptance_sha256}
{lane.lane_id}{lane.identity_state.replaceAll("_", " ")}{lane.attempted_cells}{lane.structural_passes} pass / {lane.structural_failures} fail{lane.complete_outputs_accepted} / {lane.attempted_cells}{lane.model_calls}{lane.source_summary_sha256}
-
+
-

Disclosed diagnostic limitation

-

HyperFrames compatibility finding

-
-

{evidence.diagnostic_limitations.finding}

-
-
HyperFrames
{evidence.diagnostic_limitations.hyperframes_version}
-
Control diagnostics
{evidence.diagnostic_limitations.mechanical_control_diagnostic_count}
-
Grok diagnostics
{evidence.diagnostic_limitations.grok_4_5_diagnostic_count}
-
-

Treatment: {evidence.diagnostic_limitations.compatibility_treatment}

-

Excluded auxiliary output: {evidence.diagnostic_limitations.auxiliary_description_step}

+

Blind-review boundary

+

Commit now; reveal after the vote

+
+ {[ + ["Packet public", evidence.blind_review.packet_public ? "Yes" : "No"], + ["Candidate outputs public", evidence.blind_review.candidate_outputs_public ? "Yes" : "No"], + ["Identity mapping public", evidence.blind_review.identity_mapping_public ? "Yes" : "No"], + ].map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))}
+

+ Next gate: {evidence.blind_review.next_gate.replaceAll("_", " ")}. This page contains no candidate media, per-cell identifiers, output hashes, run IDs, prompts, private paths, or model mapping. +

-
+

Frozen provenance

-

Exact contract, runner, protocol, and output hashes

+

Exact shared-input and aggregate hashes

{[ - ["Arena contract", evidence.arena_binding.arena_contract_sha256], - ["Parent contract", evidence.arena_binding.parent_contract_sha256], - ["Arena source head", evidence.arena_binding.source_head_sha], - ["Runner head", evidence.frozen_evidence.runner.head_sha], - ["Runner protocol", evidence.frozen_evidence.runner.protocol_sha256], - ["Runner prompt", evidence.frozen_evidence.runner.prompt_sha256], - ["Tool registry", evidence.frozen_evidence.tool_registry_sha256], + ["Staging manifest", evidence.frozen_evidence.staging_manifest_sha256], + ["System-eval contract", evidence.frozen_evidence.contract_sha256], + ["FineVideo release", evidence.frozen_evidence.release_sha256], ["Case pack", evidence.frozen_evidence.case_pack_sha256], ].map(([label, value]) => (
@@ -207,8 +156,8 @@ export default async function PhaseOneEvidencePage() {
{[ - ["Evidence JSON", `${ARTIFACT_ROOT}/phase-1-results.json`], - ["Evidence schema", `${ARTIFACT_ROOT}/phase-1-results.schema.json`], + ["Evidence JSON", `${ARTIFACT_ROOT}/seven-family-results.json`], + ["Evidence schema", `${ARTIFACT_ROOT}/seven-family-results.schema.json`], ["Checksums", `${ARTIFACT_ROOT}/checksums.json`], ["Public README", `${ARTIFACT_ROOT}/README.md`], ].map(([label, href]) => ( @@ -218,10 +167,25 @@ export default async function PhaseOneEvidencePage() { ))}

- Evidence artifact SHA-256: {checksums.files["phase-1-results.json"]} · Next gate: {evidence.next_gate} + Evidence artifact SHA-256: {checksums.files["seven-family-results.json"]}

+ +
+
+

Limits

+

What this evidence does not prove

+
    + {evidence.limitations.map((limitation, index) => ( +
  1. + {String(index + 1).padStart(2, "0")} + {limitation} +
    2. + ))} +
+
+
); } diff --git a/src/app/bench/composer-arena/page.tsx b/src/app/bench/composer-arena/page.tsx index 7d0a305..4a81861 100644 --- a/src/app/bench/composer-arena/page.tsx +++ b/src/app/bench/composer-arena/page.tsx @@ -9,11 +9,11 @@ const ARTIFACT_ROOT = "/bench/composer-arena/v0"; export const metadata: Metadata = { title: "Composer Archive-to-Output Arena v0 — Agentic Video Benchmarks", description: - "Non-rankable, synthetic public preview of the Composer Archive-to-Output Arena v0 contract, metrics, anonymous battle replay, and real-data handoff.", + "Non-rankable Composer Archive-to-Output Arena v0 methodology, seven-family task-success evidence, and synthetic interaction demo.", alternates: { canonical: "/bench/composer-arena" }, openGraph: { title: "Composer Archive-to-Output Arena v0", - description: "Synthetic non-rankable preview with separate controlled-agent and end-to-end system views.", + description: "Seven-family task-success evidence with zero ranks and a separate synthetic interaction demo.", type: "website", url: "https://agentic.video/bench/composer-arena", }, @@ -73,11 +73,11 @@ export default async function ComposerArenaPage() { Composer Archive-to-Output Arena v0

- A publication-candidate surface for comparing evidence-grounded editing agents and complete Composer systems. Leaderboard-like rows remain synthetic preview examples; audited Phase 1 task evidence is published separately as a dated pilot note. + A publication-candidate methodology for evidence-grounded editing agents and complete Composer systems. The dated evidence page now reports the terminal seven-family task-success aggregate; no public standings exist.

- Official ranking is disabled. The preview table and replay are synthetic, local replay judgments are excluded, and the separate Phase 1 evidence note contains no winner or preference claim. + Official ranking is disabled. There are no public score rows, the replay is a generic synthetic interaction demo, local judgments are excluded, and evaluated identities stay sealed until the founder votes.
@@ -96,16 +96,16 @@ export default async function ComposerArenaPage() {
-
+
evidence / pilot · not a leaderboard -

Phase 1 deterministic evidence is available

+

Seven-family task-success evidence is available

- Mechanical control and Grok 4.5 produced complete passing outputs; Sol recorded a tool-policy task failure, Kimi K3 was excluded for runtime/provider compatibility, and GLM 5.2 direct vision is not applicable. Zero blind battles, zero ranks, and zero human preference claims. + The frozen cohort covers seven eligible cells from seven distinct source families. It reports one public mechanical control and two identity-sealed evaluated lanes, with zero blind battles, votes, ranks, or winner.

{checksums.files["release.preview.json"]} · Arena source head: {release.arena_binding.source_head_sha}

- The command-room FineVideo selection and release marker pass for 24 cells. Phase 1 produced deterministic pilot evidence, but not a complete comparable seven-family roster or blind human review, so no official ranking exists. + The command-room FineVideo selection and release marker pass for 24 cells. Seven-family deterministic task-success evidence is terminal, but blind human review and identity reveal are still pending, so no official ranking exists.

diff --git a/src/app/bench/page.tsx b/src/app/bench/page.tsx index 20d0db1..a519c3f 100644 --- a/src/app/bench/page.tsx +++ b/src/app/bench/page.tsx @@ -130,7 +130,7 @@ export default function BenchPage() {
- Composer Archive-to-Output Arena v0 + Composer Archive-to-Output Arena v0
@@ -139,7 +139,7 @@ export default function BenchPage() {

Archive understanding to reviewable output

- Synthetic preview examples, track-separated contracts, and anonymous A/B replay sit beside a dated Phase 1 deterministic-evidence note. There is no official ranking, winner, or human preference claim. + Composer Archive-to-Output uses a separate dataset, task contract, evidence page, and publication gate. Its seven-family task-success aggregate is non-rankable and does not change SEA Broadcast ASR.

Date: Sun, 19 Jul 2026 08:58:51 -0700 Subject: [PATCH 4/4] fix: bind Arena preview to merged harness --- public/bench/composer-arena/v0/checksums.json | 8 ++++---- public/bench/composer-arena/v0/handoff.md | 2 +- public/bench/composer-arena/v0/methodology.md | 6 +++--- public/bench/composer-arena/v0/release.preview.json | 4 ++-- public/bench/composer-arena/v0/release.schema.json | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/public/bench/composer-arena/v0/checksums.json b/public/bench/composer-arena/v0/checksums.json index 4da1bbe..12683ad 100644 --- a/public/bench/composer-arena/v0/checksums.json +++ b/public/bench/composer-arena/v0/checksums.json @@ -1,11 +1,11 @@ { "algorithm": "sha256", "files": { - "release.schema.json": "a42e66add183d8e35d40fce36fa0b8ab3ba2c1e4656d7192398b0873a6f62b35", + "release.schema.json": "4d71d041bbd14a3a626f2a9e32ac6bfe47e130f6e82b3801966673fe71777b9c", "battle.schema.json": "b235183b07bf469172c79e8746275f2dd81dc30517667db5fb9a40f07c1f1572", - "release.preview.json": "ae76298c693969dd3d1b2c7c687a791b715820daf4f0570a5a9711135ae202bd", + "release.preview.json": "f061e66562b5a5c23fb4b9c4621ead0a5dc1bb70501254195f56d0c14522eb3b", "battles.preview.jsonl": "2b99e8b7f89e073ed81daeab19cd86ec984aae87f8a3ac26bc079ae66e0cb59b", - "methodology.md": "7eda2248f9b434642f796034ca1760c40c6d485b00df17fd41766210936ae3d2", - "handoff.md": "7419838696f005a0048aaaf0e13ab35e7fe50f87abded698bc7307db3bf35a10" + "methodology.md": "b76bd27c694dca8c2108b3a87139b3d908daf1cd16da405e3aad14884bcb52a0", + "handoff.md": "ed94c5c5c259bd954a6cd44467e6adbfd9e0805176509afbb63cfaa76dfd98e5" } } diff --git a/public/bench/composer-arena/v0/handoff.md b/public/bench/composer-arena/v0/handoff.md index 299d4ec..6c2d307 100644 --- a/public/bench/composer-arena/v0/handoff.md +++ b/public/bench/composer-arena/v0/handoff.md @@ -2,7 +2,7 @@ The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. The current release has empty standings and the battle file is a generic synthetic interaction demo. A real blind packet must not replace either public file before the founder votes and a separate identity reveal is authorized. -The canonical upstream contract binding is `composer-archive-to-output-arena-v0` at source head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7`, Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`, and parent system-eval contract SHA-256 `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624`. A future adapter must fail closed if any binding drifts. +The canonical upstream contract binding is `composer-archive-to-output-arena-v0` at merged `composer-mvp` source head `3dceb46215e999f4a11a564ea958f845c36af215`, Arena contract SHA-256 `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4`, and parent system-eval contract SHA-256 `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624`. A future adapter must fail closed if any binding drifts. ## Required files diff --git a/public/bench/composer-arena/v0/methodology.md b/public/bench/composer-arena/v0/methodology.md index 48f6319..a18195e 100644 --- a/public/bench/composer-arena/v0/methodology.md +++ b/public/bench/composer-arena/v0/methodology.md @@ -1,6 +1,6 @@ # Composer Archive-to-Output Arena v0 methodology -Status: **non-rankable preview**. The public standings are empty. The dated evidence page contains sanitized seven-family task-success counts, while the A/B replay contains generic synthetic interaction fixtures only. This public adapter is bound to `PixelML/agentic_video_intelligence` head `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` and Arena contract SHA-256 `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5`. +Status: **non-rankable preview**. The public standings are empty. The dated evidence page contains sanitized seven-family task-success counts, while the A/B replay contains generic synthetic interaction fixtures only. This public adapter is bound to merged `PixelML/agentic_video_intelligence` `composer-mvp` head `3dceb46215e999f4a11a564ea958f845c36af215` and Arena contract SHA-256 `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4`. ## Tracks @@ -34,8 +34,8 @@ Public-eval and private-test source families must remain disjoint. Hidden-test f ## Frozen evidence -- Arena source head: `824da0c1001662cb5a5a01e68c9ecc7d86a16bf7` -- Arena contract SHA-256: `fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5` +- Arena source head: `3dceb46215e999f4a11a564ea958f845c36af215` +- Arena contract SHA-256: `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4` - System-eval contract SHA-256: `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624` - FineVideo selection release SHA-256: `d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926` - FineVideo selection-manifest SHA-256: `39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4` diff --git a/public/bench/composer-arena/v0/release.preview.json b/public/bench/composer-arena/v0/release.preview.json index 90115b7..50b22b1 100644 --- a/public/bench/composer-arena/v0/release.preview.json +++ b/public/bench/composer-arena/v0/release.preview.json @@ -10,8 +10,8 @@ "display_label": "non-rankable preview", "arena_binding": { "arena_id": "composer-archive-to-output-arena-v0", - "source_head_sha": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7", - "arena_contract_sha256": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5", + "source_head_sha": "3dceb46215e999f4a11a564ea958f845c36af215", + "arena_contract_sha256": "8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4", "parent_contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" }, "publication": { diff --git a/public/bench/composer-arena/v0/release.schema.json b/public/bench/composer-arena/v0/release.schema.json index 0d30e49..cad2a8a 100644 --- a/public/bench/composer-arena/v0/release.schema.json +++ b/public/bench/composer-arena/v0/release.schema.json @@ -224,8 +224,8 @@ "required": ["arena_id", "source_head_sha", "arena_contract_sha256", "parent_contract_sha256"], "properties": { "arena_id": { "const": "composer-archive-to-output-arena-v0" }, - "source_head_sha": { "const": "824da0c1001662cb5a5a01e68c9ecc7d86a16bf7" }, - "arena_contract_sha256": { "const": "fb3f35346af22941d4476d390a1f2929d50653f9f50ee23cef8d7b2b1ad3ecc5" }, + "source_head_sha": { "const": "3dceb46215e999f4a11a564ea958f845c36af215" }, + "arena_contract_sha256": { "const": "8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4" }, "parent_contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" } } },