diff --git a/package-lock.json b/package-lock.json index d088b57..4c67c1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", + "ajv": "^8.17.1", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", @@ -418,6 +419,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@eslint/eslintrc/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/@eslint/js": { "version": "9.39.2", "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", @@ -2139,16 +2164,16 @@ } }, "node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -3382,6 +3407,30 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -3497,6 +3546,23 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-uri": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -4469,9 +4535,9 @@ "license": "MIT" }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, @@ -5503,6 +5569,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.11", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", diff --git a/package.json b/package.json index 96d7236..e74c361 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "version": "0.1.0", "private": true, "scripts": { - "check:bench": "node scripts/check-bench-route.mjs", + "check:arena": "node scripts/check-composer-arena.mjs", + "check:bench": "node scripts/check-bench-route.mjs && node scripts/check-composer-arena.mjs", "dev": "next dev", "build": "next build", "start": "next start", @@ -21,6 +22,7 @@ "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", + "ajv": "^8.17.1", "eslint": "^9", "eslint-config-next": "16.1.6", "tailwindcss": "^4", diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md new file mode 100644 index 0000000..9329ffe --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/README.md @@ -0,0 +1,14 @@ +# Composer Arena seven-family task-success evidence — 2026-07-19 + +This is a sanitized deterministic-evidence aggregate over seven eligible FineVideo public-eval cells from seven distinct source families. It is not a leaderboard, preference result, or publication approval. + +The public packet reports only cohort counts and stable evidence hashes: + +- mechanical control: 6/7 structural passes and 5/7 complete outputs accepted; +- evaluated lane A: 2/7 complete outputs accepted; +- evaluated lane B: 4/7 complete outputs accepted; +- zero blind battles, votes, ranks, or winner. + +Evaluated model identities remain sealed pending the founder's blind vote. The real blind packet, candidate media, per-cell identifiers, output hashes, prompts, run identifiers, private paths, and identity mapping are not included here. + +Deterministic task success and complete-output acceptance do not measure editorial preference. SEA Broadcast ASR remains a separate benchmark suite with independent sources, rights, scoring, and publication gates. diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json new file mode 100644 index 0000000..2c50d72 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/checksums.json @@ -0,0 +1,8 @@ +{ + "algorithm": "sha256", + "files": { + "seven-family-results.json": "ed2cba9fb604a7d2db70b862fd760c5287e55b21d3b4cc6aeb8415285fa57d1f", + "seven-family-results.schema.json": "a26f151cb7102294f8e7a0712031bd6a2462b1561568901edc37f97e586bade3", + "README.md": "0630d7415cded4039c6fde74101610a6b746cd0556fa78807c5e2fff51b6a61a" + } +} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json new file mode 100644 index 0000000..4ae7149 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.json @@ -0,0 +1,93 @@ +{ + "schema_version": 1, + "evidence_id": "composer-arena-seven-family-task-success-2026-07-19", + "evidence_date": "2026-07-19", + "status": "seven-family-terminal-task-success-publication-not-approved", + "data_class": "sanitized_task_success_aggregate", + "publication": { + "surface": "evidence/pilot", + "publishable": false, + "state": "insufficient_evidence", + "leaderboard": false, + "winner": null, + "rank_count": 0, + "human_preference_claims": false + }, + "vote_provenance": { + "collection_mode": "no_eligible_votes", + "crowdsourced": false, + "blind_battles": 0, + "real_human_votes": 0, + "resolved_battle_groups": 0 + }, + "frozen_evidence": { + "dataset_repo_id": "HuggingFaceFV/finevideo", + "dataset_revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", + "staging_manifest_sha256": "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8", + "contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", + "release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + "case_pack_sha256": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203" + }, + "cohort": { + "eligible_cells": 7, + "distinct_source_families": 7, + "favorable_selection": false, + "source_family_disjoint": true + }, + "task_success": { + "separation_statement": "Deterministic task success and complete-output acceptance are system evidence only; they never create a preference score, rank, or winner.", + "lanes": [ + { + "lane_id": "mechanical-control", + "classification": "control", + "identity_state": "public_control", + "attempted_cells": 7, + "structural_passes": 6, + "structural_failures": 1, + "complete_outputs_accepted": 5, + "complete_output_failures": 2, + "model_calls": 0, + "source_summary_sha256": "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57" + }, + { + "lane_id": "evaluated-lane-a", + "classification": "evaluated_system", + "identity_state": "sealed_pending_founder_vote", + "attempted_cells": 7, + "structural_passes": 2, + "structural_failures": 5, + "complete_outputs_accepted": 2, + "complete_output_failures": 5, + "model_calls": 7, + "source_summary_sha256": "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a" + }, + { + "lane_id": "evaluated-lane-b", + "classification": "evaluated_system", + "identity_state": "sealed_pending_founder_vote", + "attempted_cells": 7, + "structural_passes": 4, + "structural_failures": 3, + "complete_outputs_accepted": 4, + "complete_output_failures": 3, + "model_calls": 7, + "source_summary_sha256": "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456" + } + ] + }, + "blind_review": { + "packet_public": false, + "candidate_outputs_public": false, + "identity_mapping_public": false, + "founder_vote_recorded": false, + "next_gate": "founder_blind_vote_then_separate_identity_reveal" + }, + "limitations": [ + "Evaluated lane identities are deliberately sealed until the founder records the blind vote.", + "The public aggregate contains no per-cell identifiers, output hashes, candidate media, prompts, run identifiers, or identity mapping.", + "Complete-output acceptance measures deterministic delivery gates, not editorial quality or human preference.", + "The controlled-agent track remains blocked because the released evidence pack contains no transcript, OCR, or semantic events.", + "One mechanical-control cell failed structurally and one rendered control output failed the opening-black gate; neither failure was repaired inside the frozen cohort.", + "No official ranking, winner, governed vote backend, or public blind packet exists." + ] +} diff --git a/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json new file mode 100644 index 0000000..8942372 --- /dev/null +++ b/public/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json @@ -0,0 +1,150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/evidence/pilot/2026-07-19/seven-family-results.schema.json", + "title": "Composer Arena seven-family public task-success evidence", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "evidence_id", + "evidence_date", + "status", + "data_class", + "publication", + "vote_provenance", + "frozen_evidence", + "cohort", + "task_success", + "blind_review", + "limitations" + ], + "properties": { + "schema_version": { "const": 1 }, + "evidence_id": { "const": "composer-arena-seven-family-task-success-2026-07-19" }, + "evidence_date": { "const": "2026-07-19" }, + "status": { "const": "seven-family-terminal-task-success-publication-not-approved" }, + "data_class": { "const": "sanitized_task_success_aggregate" }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["surface", "publishable", "state", "leaderboard", "winner", "rank_count", "human_preference_claims"], + "properties": { + "surface": { "const": "evidence/pilot" }, + "publishable": { "const": false }, + "state": { "const": "insufficient_evidence" }, + "leaderboard": { "const": false }, + "winner": { "type": "null" }, + "rank_count": { "const": 0 }, + "human_preference_claims": { "const": false } + } + }, + "vote_provenance": { + "type": "object", + "additionalProperties": false, + "required": ["collection_mode", "crowdsourced", "blind_battles", "real_human_votes", "resolved_battle_groups"], + "properties": { + "collection_mode": { "const": "no_eligible_votes" }, + "crowdsourced": { "const": false }, + "blind_battles": { "const": 0 }, + "real_human_votes": { "const": 0 }, + "resolved_battle_groups": { "const": 0 } + } + }, + "frozen_evidence": { + "type": "object", + "additionalProperties": false, + "required": ["dataset_repo_id", "dataset_revision", "staging_manifest_sha256", "contract_sha256", "release_sha256", "case_pack_sha256"], + "properties": { + "dataset_repo_id": { "const": "HuggingFaceFV/finevideo" }, + "dataset_revision": { "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" }, + "staging_manifest_sha256": { "const": "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8" }, + "contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" }, + "release_sha256": { "const": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926" }, + "case_pack_sha256": { "const": "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203" } + } + }, + "cohort": { + "type": "object", + "additionalProperties": false, + "required": ["eligible_cells", "distinct_source_families", "favorable_selection", "source_family_disjoint"], + "properties": { + "eligible_cells": { "const": 7 }, + "distinct_source_families": { "const": 7 }, + "favorable_selection": { "const": false }, + "source_family_disjoint": { "const": true } + } + }, + "task_success": { + "type": "object", + "additionalProperties": false, + "required": ["separation_statement", "lanes"], + "properties": { + "separation_statement": { "type": "string", "minLength": 1 }, + "lanes": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + { "$ref": "#/$defs/controlLane" }, + { "$ref": "#/$defs/evaluatedLaneA" }, + { "$ref": "#/$defs/evaluatedLaneB" } + ] + } + } + }, + "blind_review": { + "type": "object", + "additionalProperties": false, + "required": ["packet_public", "candidate_outputs_public", "identity_mapping_public", "founder_vote_recorded", "next_gate"], + "properties": { + "packet_public": { "const": false }, + "candidate_outputs_public": { "const": false }, + "identity_mapping_public": { "const": false }, + "founder_vote_recorded": { "const": false }, + "next_gate": { "const": "founder_blind_vote_then_separate_identity_reveal" } + } + }, + "limitations": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + }, + "$defs": { + "laneBase": { + "type": "object", + "additionalProperties": false, + "required": ["lane_id", "classification", "identity_state", "attempted_cells", "structural_passes", "structural_failures", "complete_outputs_accepted", "complete_output_failures", "model_calls", "source_summary_sha256"], + "properties": { + "lane_id": { "type": "string", "minLength": 1 }, + "classification": { "enum": ["control", "evaluated_system"] }, + "identity_state": { "enum": ["public_control", "sealed_pending_founder_vote"] }, + "attempted_cells": { "const": 7 }, + "structural_passes": { "type": "integer", "minimum": 0, "maximum": 7 }, + "structural_failures": { "type": "integer", "minimum": 0, "maximum": 7 }, + "complete_outputs_accepted": { "type": "integer", "minimum": 0, "maximum": 7 }, + "complete_output_failures": { "type": "integer", "minimum": 0, "maximum": 7 }, + "model_calls": { "type": "integer", "minimum": 0 }, + "source_summary_sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } + }, + "controlLane": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "mechanical-control" }, "classification": { "const": "control" }, "identity_state": { "const": "public_control" }, "structural_passes": { "const": 6 }, "structural_failures": { "const": 1 }, "complete_outputs_accepted": { "const": 5 }, "complete_output_failures": { "const": 2 }, "model_calls": { "const": 0 }, "source_summary_sha256": { "const": "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57" } } } + ] + }, + "evaluatedLaneA": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "evaluated-lane-a" }, "classification": { "const": "evaluated_system" }, "identity_state": { "const": "sealed_pending_founder_vote" }, "structural_passes": { "const": 2 }, "structural_failures": { "const": 5 }, "complete_outputs_accepted": { "const": 2 }, "complete_output_failures": { "const": 5 }, "model_calls": { "const": 7 }, "source_summary_sha256": { "const": "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a" } } } + ] + }, + "evaluatedLaneB": { + "allOf": [ + { "$ref": "#/$defs/laneBase" }, + { "properties": { "lane_id": { "const": "evaluated-lane-b" }, "classification": { "const": "evaluated_system" }, "identity_state": { "const": "sealed_pending_founder_vote" }, "structural_passes": { "const": 4 }, "structural_failures": { "const": 3 }, "complete_outputs_accepted": { "const": 4 }, "complete_output_failures": { "const": 3 }, "model_calls": { "const": 7 }, "source_summary_sha256": { "const": "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456" } } } + ] + } + } +} diff --git a/public/bench/composer-arena/v0/battle.schema.json b/public/bench/composer-arena/v0/battle.schema.json new file mode 100644 index 0000000..d7b94fe --- /dev/null +++ b/public/bench/composer-arena/v0/battle.schema.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/v0/battle.schema.json", + "title": "Composer Arena anonymous battle replay", + "type": "object", + "additionalProperties": false, + "required": ["battle_id", "track", "task_slice", "prompt", "source", "left", "right"], + "properties": { + "battle_id": { "type": "string", "minLength": 1 }, + "track": { "enum": ["controlled-agent", "end-to-end-system"] }, + "task_slice": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string", "minLength": 1 }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["fixture_id", "source_family_id", "revision", "rankable"], + "properties": { + "fixture_id": { "type": "string", "minLength": 1 }, + "source_family_id": { "type": "string", "minLength": 1 }, + "revision": { "type": "string", "minLength": 1 }, + "rankable": { "type": "boolean" } + } + }, + "left": { "$ref": "#/$defs/candidate" }, + "right": { "$ref": "#/$defs/candidate" } + }, + "$defs": { + "candidate": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidate_id", + "model", + "provider", + "version", + "output_title", + "output_summary", + "evidence" + ], + "properties": { + "candidate_id": { "type": "string", "minLength": 1 }, + "model": { "type": "string", "minLength": 1 }, + "provider": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "output_title": { "type": "string", "minLength": 1 }, + "output_summary": { "type": "string", "minLength": 1 }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "href"], + "properties": { + "label": { "type": "string", "minLength": 1 }, + "href": { "type": "string", "pattern": "^/bench/composer-arena/v0/" } + } + } + } + } + } + } +} diff --git a/public/bench/composer-arena/v0/battles.preview.jsonl b/public/bench/composer-arena/v0/battles.preview.jsonl new file mode 100644 index 0000000..6ebc7a4 --- /dev/null +++ b/public/bench/composer-arena/v0/battles.preview.jsonl @@ -0,0 +1,2 @@ +{"battle_id":"preview-controlled-001","track":"controlled-agent","task_slice":"temporal-ordering","prompt":"Create an evidence-grounded edit plan that opens on the constraint, preserves source chronology, and cites every selected window.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-chronology","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"demo-controlled-alpha","model":"Synthetic demo Alpha","provider":"Not applicable","version":"fixture-alpha-v1","output_title":"Output A","output_summary":"Three-window synthetic plan with explicit evidence IDs, chronological ordering, and a short unsupported transition note.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Methodology","href":"/bench/composer-arena/v0/methodology.md"}]},"right":{"candidate_id":"demo-controlled-beta","model":"Synthetic demo Beta","provider":"Not applicable","version":"fixture-beta-v1","output_title":"Output B","output_summary":"Three-window synthetic plan with complete evidence IDs and valid ranges; the opening constraint follows a brief setup beat.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Release fixture","href":"/bench/composer-arena/v0/release.preview.json"}]}} +{"battle_id":"preview-system-001","track":"end-to-end-system","task_slice":"reviewable-delivery","prompt":"Produce a structurally valid 45-second review cut with synchronized source windows, no blocking gaps, and a provenance-complete delivery record.","source":{"fixture_id":"synthetic-archive-v1","source_family_id":"synthetic-family-delivery","revision":"synthetic-archive-v1","rankable":false},"left":{"candidate_id":"demo-system-alpha","model":"Synthetic demo Alpha","provider":"Not applicable","version":"fixture-alpha-v1","output_title":"Output A","output_summary":"Synthetic 45-second timeline with complete provenance and one non-blocking safe-area warning.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Schema","href":"/bench/composer-arena/v0/release.schema.json"}]},"right":{"candidate_id":"demo-system-beta","model":"Synthetic demo Beta","provider":"Not applicable","version":"fixture-beta-v1","output_title":"Output B","output_summary":"Synthetic synchronized timeline with a clean structural gate and an omitted source-family label.","evidence":[{"label":"Synthetic battle record","href":"/bench/composer-arena/v0/battles.preview.jsonl"},{"label":"Handoff contract","href":"/bench/composer-arena/v0/handoff.md"}]}} diff --git a/public/bench/composer-arena/v0/checksums.json b/public/bench/composer-arena/v0/checksums.json new file mode 100644 index 0000000..12683ad --- /dev/null +++ b/public/bench/composer-arena/v0/checksums.json @@ -0,0 +1,11 @@ +{ + "algorithm": "sha256", + "files": { + "release.schema.json": "4d71d041bbd14a3a626f2a9e32ac6bfe47e130f6e82b3801966673fe71777b9c", + "battle.schema.json": "b235183b07bf469172c79e8746275f2dd81dc30517667db5fb9a40f07c1f1572", + "release.preview.json": "f061e66562b5a5c23fb4b9c4621ead0a5dc1bb70501254195f56d0c14522eb3b", + "battles.preview.jsonl": "2b99e8b7f89e073ed81daeab19cd86ec984aae87f8a3ac26bc079ae66e0cb59b", + "methodology.md": "b76bd27c694dca8c2108b3a87139b3d908daf1cd16da405e3aad14884bcb52a0", + "handoff.md": "ed94c5c5c259bd954a6cd44467e6adbfd9e0805176509afbb63cfaa76dfd98e5" + } +} diff --git a/public/bench/composer-arena/v0/handoff.md b/public/bench/composer-arena/v0/handoff.md new file mode 100644 index 0000000..6c2d307 --- /dev/null +++ b/public/bench/composer-arena/v0/handoff.md @@ -0,0 +1,64 @@ +# Composer Arena real-data handoff + +The route reads `release.preview.json` and `battles.preview.jsonl` from this directory at build time. The current release has empty standings and the battle file is a generic synthetic interaction demo. A real blind packet must not replace either public file before the founder votes and a separate identity reveal is authorized. + +The canonical upstream contract binding is `composer-archive-to-output-arena-v0` at merged `composer-mvp` source head `3dceb46215e999f4a11a564ea958f845c36af215`, Arena contract SHA-256 `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4`, and parent system-eval contract SHA-256 `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624`. A future adapter must fail closed if any binding drifts. + +## Required files + +1. `release.preview.json` — one release object validated against `release.schema.json`. +2. `battles.preview.jsonl` — one JSON object per replayable battle, with each line validated against `battle.schema.json`. +3. `checksums.json` — raw SHA-256 values for the schema, release, battles, methodology, and this handoff. +4. `methodology.md` — frozen methodology, estimator, vote policy, source/rights boundary, contamination policy, and limitations. + +Before a rankable release, change `release_mode` to `governed_rankable_release` and `official_ranking_enabled` to `true` only after the already-green FineVideo release marker is joined by real runner records, a governed vote backend when votes are included, a frozen statistical policy, and command-room review. Synthetic or local/demo data remains `synthetic_non_rankable_preview`; sanitized task-success evidence remains separate from standings. + +The canonical upstream publication vocabulary is `ranked`, `tied`, and `insufficient_evidence`; public results have `scope: independent_track_results_only`. `controlled-semantic-text-evidence-v0` and `end-to-end-visual-orchestration-v0` remain separate, and a combined winner is forbidden. + +SEA Broadcast ASR remains a separate benchmark suite with its own rights, source, scoring, and publication contracts. Composer Arena artifacts must not change, absorb, or reinterpret SEA-ASR state. + +## Release rows + +Every track row must provide: + +- `standing`: `ranked`, `tied`, or `insufficient_evidence`; +- `display_rank`: ordinal text only for a governed publishable release; synthetic/pilot evidence uses `—`; +- model, provider, version, and evaluation date; +- Arena score, two-element 95% confidence interval, and battle count; +- deterministic task success and failure rate in `[0, 1]`; +- p50 latency in milliseconds; +- input, output, cache-read, and cache-write token totals; +- estimated USD cost per run; +- at least one per-task slice with task ID, label, success rate, and battle count. + +`controlled-agent` and `end-to-end-system` must remain separate track objects bound to the exact canonical track IDs above. Exploratory capability cards must use `status: non_rankable` and cannot be folded into Arena score. + +## Battle JSONL contract + +Each line must be a self-contained object with: + +- `battle_id`, `track`, `task_slice`, and the public-safe task prompt; +- a `source` object containing fixture/cell identity, source-family ID, immutable revision, and `rankable`; +- `left` and `right` candidates containing a stable candidate ID, hidden model/provider/version identity, public output title/summary, and one or more provenance/evidence links. + +The browser may swap left/right display order. Local demo choices are exactly `left`, `right`, `tie`, `both_bad`, and `abstain`. The public client reveals only generic demo identities after a choice and sends no vote request. + +## Private blind packet boundary + +- Keep the real packet, candidate media, per-cell identifiers, output hashes, prompts, run identifiers, and model mapping outside `public/` until the founder records a vote. +- Public pre-vote evidence may contain only aggregate task-success counts, shared-input hashes, and identity-sealed source-summary commitments. +- Do not expose a real candidate through static JSON, HTML source, build output, evidence links, checksums, or downloadable artifacts. +- Publish any model-identity reveal as a separate, dated post-vote artifact so the pre-vote evidence remains immutable. + +## Promotion gates + +- FineVideo repo and revision match the frozen contract. +- Raw contract and selection-release SHA-256 values match reviewed markers. +- Rankable runner/runtime records exist and validate; a green source-release gate alone is not a model-result release. +- Public-eval and private-test source families are disjoint. +- No hidden test data, customer/GMA data, credentials, signed URLs, or raw reasoning traces enter public artifacts. +- Every model route passes exact-provider/model/version preflight with no fallback substitution. +- The statistical exporter emits ties and insufficient-evidence states rather than forcing ranks. +- Deterministic seven-family task-success evidence creates no battle or Bradley-Terry observation without a private comparable packet and a governed human judgment. +- Official votes, if any, come only from a governed backend with a versioned inclusion policy. +- Schema validation, checksum verification, lint, build, route contract checks, and HTTP/UI smoke are green. diff --git a/public/bench/composer-arena/v0/methodology.md b/public/bench/composer-arena/v0/methodology.md new file mode 100644 index 0000000..a18195e --- /dev/null +++ b/public/bench/composer-arena/v0/methodology.md @@ -0,0 +1,60 @@ +# Composer Archive-to-Output Arena v0 methodology + +Status: **non-rankable preview**. The public standings are empty. The dated evidence page contains sanitized seven-family task-success counts, while the A/B replay contains generic synthetic interaction fixtures only. This public adapter is bound to merged `PixelML/agentic_video_intelligence` `composer-mvp` head `3dceb46215e999f4a11a564ea958f845c36af215` and Arena contract SHA-256 `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4`. + +## Tracks + +- `controlled-semantic-text-evidence-v0` / `controlled-agent` requires one genuine, hash-identical transcript/OCR/semantic-event artifact. That artifact is unavailable, so the track remains blocked before model execution. +- `end-to-end-visual-orchestration-v0` / `end-to-end-system` has terminal seven-family deterministic task-success evidence for one public control and two identity-sealed evaluated lanes. It has no public preference result. +- The two tracks are reported separately and never collapsed into one leaderboard. + +## Standing and uncertainty + +The canonical Arena estimator is Bradley-Terry logistic maximum likelihood over governed resolved battles. Uncertainty uses 2,000 deterministic source-family cluster-bootstrap replicates at 95% confidence. A future governed release must declare its score calibration and reference baseline; this preview publishes no score, interval, or model standing. + +An ordinal standing is forbidden when the frozen battle, seven-source-family, human-judge, complete-roster, connectivity, bootstrap, contamination, or position-bias gates are not met. `tied` means a pairwise 95% interval contains zero; `insufficient_evidence` means one or more gates failed. The current release has no rows and remains `publishable=false`, `rank_count=0`, `winner=null`, and non-rankable. + +Deterministic task success is computed independently from preference battles. Failure rate retains explicit terminal failures in the denominator. Latency, token totals, and estimated cost use frozen run records; they never disappear because a run failed. + +## Anonymous battle replay + +The replay is a generic synthetic interaction demo. It records `A is better`, `B is better`, `Tie`, `Both bad`, or `Abstain`; A/B order can be swapped before judgment. The choice is stored only in React state for the current browser view: there is no API request, persistence, identity, anti-abuse control, or official ranking effect. + +The real blind packet, candidate media, per-cell identifiers, output hashes, and identity mapping remain private until the founder votes. They are not represented by `battles.preview.jsonl`, and a static public artifact must never be treated as an identity-secrecy boundary. + +Any future governed vote backend requires a reviewed sampling policy, replay identity, duplicate/abuse controls, immutable battle and output hashes, model reveal rules, audit export, deletion/privacy handling, and a versioned policy that says exactly which votes enter an official release. + +## FineVideo source and attribution + +The rankable source contract freezes `HuggingFaceFV/finevideo` at immutable revision `84c74091e1c6ee7a5dffabfafb5c9033e4718883`. The command-room-audited selection manifest contains 24 `public-eval` source families. Those selected cells carry the `CC-BY` claim from FineVideo publisher metadata and dataset terms, plus source-specific creator attribution text. + +Public examples must retain the exact per-cell attribution and provenance chain. This preview fetches or re-hosts no media and contains no real evaluated output. It does not broaden a per-cell rights record into a blanket claim about unrelated FineVideo content. + +Public-eval and private-test source families must remain disjoint. Hidden-test families, annotations, prompts, and failure history stay private and are never exposed through the public route or battle replay. + +## Frozen evidence + +- Arena source head: `3dceb46215e999f4a11a564ea958f845c36af215` +- Arena contract SHA-256: `8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4` +- System-eval contract SHA-256: `33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624` +- FineVideo selection release SHA-256: `d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926` +- FineVideo selection-manifest SHA-256: `39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4` +- Seven-family staging-manifest SHA-256: `0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8` +- Seven-family case-pack SHA-256: `74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203` + +The separate `FINEVIDEO_RELEASED` marker exists and the official release check passes for 24 cells with the contract and release hashes above. Seven eligible cells from seven distinct families completed the deterministic protocol with no favorable selection. The public aggregate still publishes no candidate output, preference score, identity mapping, or rank. + +## Contamination and privacy boundaries + +No evaluated model may judge its own or another baseline's output. Training overlap, prompt leakage, source-family reuse, hidden-test tuning, or model substitution disables official rankability. Human editorial review is blinded, uses two reviewers plus adjudication, and remains separate from deterministic scoring. + +Public exports may contain aggregate metrics, public-safe artifact hashes, required FineVideo attribution, and reviewed examples only. They must never contain credentials, signed URLs, provider headers, hidden annotations, raw reasoning traces, unnecessary personal metadata, customer data, or GMA data. + +## Known limitations + +- No preview score, interval, cost, model rank, or winner is published. +- Seven-family task-success counts are deterministic system evidence, not editorial preference. +- Evaluated lane identities remain sealed pending the founder's blind vote. +- The public surface does not implement an official vote backend. +- The frozen system-eval contract currently uses one repetition per real cell; reliability claims need a reviewed repeated-run extension. +- Editorial preference and long-horizon recovery remain exploratory, non-rankable capability views. diff --git a/public/bench/composer-arena/v0/release.preview.json b/public/bench/composer-arena/v0/release.preview.json new file mode 100644 index 0000000..50b22b1 --- /dev/null +++ b/public/bench/composer-arena/v0/release.preview.json @@ -0,0 +1,107 @@ +{ + "schema_version": 1, + "contract_version": "composer-arena-public-release-v0.1", + "release_id": "composer-arena-v0-synthetic-preview-2026-07-19", + "generated_at": "2026-07-19T12:00:00Z", + "release_mode": "synthetic_non_rankable_preview", + "data_class": "synthetic", + "official_ranking_enabled": false, + "official_vote_backend": "not_configured", + "display_label": "non-rankable preview", + "arena_binding": { + "arena_id": "composer-archive-to-output-arena-v0", + "source_head_sha": "3dceb46215e999f4a11a564ea958f845c36af215", + "arena_contract_sha256": "8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4", + "parent_contract_sha256": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" + }, + "publication": { + "publishable": false, + "state": "insufficient_evidence", + "scope": "independent_track_results_only", + "winner": null, + "rank_count": 0 + }, + "vote_provenance": { + "collection_mode": "synthetic_fixture", + "crowdsourced": false, + "real_human_votes": 0 + }, + "dataset": { + "repo_id": "HuggingFaceFV/finevideo", + "revision": "84c74091e1c6ee7a5dffabfafb5c9033e4718883", + "license_claim": "CC-BY", + "license_scope": "The 24 command-room-audited public-eval cells carry CC-BY publisher metadata in the frozen selection manifest. Each real public example must retain its source-specific attribution text.", + "attribution": "FineVideo, HuggingFaceFV/finevideo at immutable revision 84c74091e1c6ee7a5dffabfafb5c9033e4718883. Source-specific creator attribution is retained per released cell.", + "finevideo_release_sha256": "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + "selection_manifest_sha256": "39d34f73c3708f62bf4bc0f25dae721f844821d3c75c0179e4cedcc183a71ba4", + "public_eval_source_family_count": 24, + "split_policy": { + "source_family_disjoint": true, + "public_eval": "Only command-room-released source families may appear in public evaluation.", + "private_test": "Private-test families and annotations remain unpublished and must never appear in prompts or local demo artifacts.", + "hidden_test_published": false + } + }, + "evaluation_policy": { + "ranking_method": "Bradley-Terry logistic maximum likelihood", + "confidence_level": 0.95, + "bootstrap_replicates": 2000, + "minimum_resolved_battles_per_pair_overall": 4, + "position_bias_decisive_presentations": 20, + "tie_rule": "Show a tie when the frozen comparison cannot establish a separated standing at 95% confidence.", + "insufficient_evidence_rule": "Do not assign an ordinal standing below the minimum battle count or when required evidence is incomplete.", + "local_vote_policy": "Replay judgments are browser-local demonstrations. They are not transmitted, persisted, or included in official standings.", + "contamination_limit": "No evaluated model may judge any baseline output. Training overlap, prompt leakage, source-family reuse, or hidden-test tuning disables official rankability.", + "privacy_limit": "Public exports contain aggregate metrics, public-safe hashes, required attribution, and reviewed examples only; never credentials, signed URLs, hidden annotations, raw reasoning traces, customer data, or GMA data." + }, + "tracks": [ + { + "id": "controlled-agent", + "canonical_track_id": "controlled-semantic-text-evidence-v0", + "title": "Controlled agent", + "description": "No public standing exists. The released evidence pack contains no transcript, OCR, or semantic events, so this track remains blocked before model execution.", + "rows": [] + }, + { + "id": "end-to-end-system", + "canonical_track_id": "end-to-end-visual-orchestration-v0", + "title": "End-to-end system", + "description": "Seven-family deterministic task-success evidence exists, but no governed blind preference result or public standing exists.", + "rows": [] + } + ], + "exploratory_capabilities": [ + { + "id": "direct-perception", + "title": "Direct visual perception", + "status": "non_rankable", + "summary": "Can a route inspect released frames directly rather than reason only over frozen derived evidence?", + "evidence_scope": "Capability declaration and preflight only; no capability statement is converted into rank.", + "limitation": "Unavailable direct perception is recorded as not applicable rather than an editing-quality failure." + }, + { + "id": "editorial-preference", + "title": "Editorial preference", + "status": "non_rankable", + "summary": "Blind human review of coherence, opening strength, cut naturalness, and caption readability.", + "evidence_scope": "A private, identity-sealed packet and a recorded human judgment are required before any reveal.", + "limitation": "Deterministic task success is not editorial preference and never creates a winner." + }, + { + "id": "long-horizon-recovery", + "title": "Long-horizon recovery", + "status": "non_rankable", + "summary": "Recovery from tool failures, stale state, and interrupted editing turns.", + "evidence_scope": "Exploratory fault-injection notes only.", + "limitation": "The v0 public contract does not freeze a comparable failure-injection distribution." + } + ], + "known_limitations": [ + "No public Arena score, confidence interval, battle count, model rank, cost comparison, or winner exists.", + "Seven-family task-success evidence is deterministic system evidence, not a preference result.", + "The controlled-agent track remains blocked because the released evidence pack has no transcript, OCR, or semantic events.", + "The local replay is a synthetic interaction demo with generic candidates and has no governed identity, sampling, persistence, or ranking effect.", + "The real blind packet, candidate outputs, and model identity mapping remain private until the founder records a vote.", + "One repetition per real cell is frozen; reliability claims require a reviewed repeated-run extension." + ] +} diff --git a/public/bench/composer-arena/v0/release.schema.json b/public/bench/composer-arena/v0/release.schema.json new file mode 100644 index 0000000..cad2a8a --- /dev/null +++ b/public/bench/composer-arena/v0/release.schema.json @@ -0,0 +1,268 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentic.video/bench/composer-arena/v0/release.schema.json", + "title": "Composer Archive-to-Output Arena public release", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "contract_version", + "release_id", + "generated_at", + "release_mode", + "data_class", + "official_ranking_enabled", + "official_vote_backend", + "display_label", + "arena_binding", + "publication", + "vote_provenance", + "dataset", + "evaluation_policy", + "tracks", + "exploratory_capabilities", + "known_limitations" + ], + "properties": { + "schema_version": { "const": 1 }, + "contract_version": { "const": "composer-arena-public-release-v0.1" }, + "release_id": { "type": "string", "minLength": 1 }, + "generated_at": { "type": "string", "format": "date-time" }, + "release_mode": { + "enum": ["synthetic_non_rankable_preview", "governed_rankable_release"] + }, + "data_class": { "enum": ["rankable", "exploratory", "synthetic"] }, + "official_ranking_enabled": { "type": "boolean" }, + "official_vote_backend": { "enum": ["not_configured", "governed_v1"] }, + "display_label": { "type": "string", "minLength": 1 }, + "arena_binding": { "$ref": "#/$defs/arenaBinding" }, + "publication": { "$ref": "#/$defs/publication" }, + "vote_provenance": { "$ref": "#/$defs/voteProvenance" }, + "dataset": { "$ref": "#/$defs/dataset" }, + "evaluation_policy": { "$ref": "#/$defs/evaluationPolicy" }, + "tracks": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "items": { "$ref": "#/$defs/track" } + }, + "exploratory_capabilities": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/exploratoryCapability" } + }, + "known_limitations": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "dataset": { + "type": "object", + "additionalProperties": false, + "required": [ + "repo_id", + "revision", + "license_claim", + "license_scope", + "attribution", + "finevideo_release_sha256", + "selection_manifest_sha256", + "public_eval_source_family_count", + "split_policy" + ], + "properties": { + "repo_id": { "const": "HuggingFaceFV/finevideo" }, + "revision": { + "const": "84c74091e1c6ee7a5dffabfafb5c9033e4718883" + }, + "license_claim": { "const": "CC-BY" }, + "license_scope": { "type": "string", "minLength": 1 }, + "attribution": { "type": "string", "minLength": 1 }, + "finevideo_release_sha256": { "$ref": "#/$defs/sha256" }, + "selection_manifest_sha256": { "$ref": "#/$defs/sha256" }, + "public_eval_source_family_count": { "type": "integer", "minimum": 1 }, + "split_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_family_disjoint", + "public_eval", + "private_test", + "hidden_test_published" + ], + "properties": { + "source_family_disjoint": { "const": true }, + "public_eval": { "type": "string", "minLength": 1 }, + "private_test": { "type": "string", "minLength": 1 }, + "hidden_test_published": { "const": false } + } + } + } + }, + "evaluationPolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "ranking_method", + "confidence_level", + "bootstrap_replicates", + "minimum_resolved_battles_per_pair_overall", + "position_bias_decisive_presentations", + "tie_rule", + "insufficient_evidence_rule", + "local_vote_policy", + "contamination_limit", + "privacy_limit" + ], + "properties": { + "ranking_method": { "const": "Bradley-Terry logistic maximum likelihood" }, + "confidence_level": { "const": 0.95 }, + "bootstrap_replicates": { "const": 2000 }, + "minimum_resolved_battles_per_pair_overall": { "const": 4 }, + "position_bias_decisive_presentations": { "const": 20 }, + "tie_rule": { "type": "string", "minLength": 1 }, + "insufficient_evidence_rule": { "type": "string", "minLength": 1 }, + "local_vote_policy": { "type": "string", "minLength": 1 }, + "contamination_limit": { "type": "string", "minLength": 1 }, + "privacy_limit": { "type": "string", "minLength": 1 } + } + }, + "track": { + "type": "object", + "additionalProperties": false, + "required": ["id", "canonical_track_id", "title", "description", "rows"], + "properties": { + "id": { "enum": ["controlled-agent", "end-to-end-system"] }, + "canonical_track_id": { "enum": ["controlled-semantic-text-evidence-v0", "end-to-end-visual-orchestration-v0"] }, + "title": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "rows": { + "type": "array", + "minItems": 0, + "items": { "$ref": "#/$defs/row" } + } + } + }, + "row": { + "type": "object", + "additionalProperties": false, + "required": [ + "standing", + "display_rank", + "model", + "provider", + "version", + "evaluation_date", + "arena_score", + "arena_score_ci95", + "battle_count", + "deterministic_task_success", + "failure_rate", + "latency_ms_p50", + "tokens", + "estimated_cost_usd_per_run", + "per_task_slices" + ], + "properties": { + "standing": { "enum": ["ranked", "tied", "insufficient_evidence"] }, + "display_rank": { "type": "string", "minLength": 1 }, + "model": { "type": "string", "minLength": 1 }, + "provider": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "evaluation_date": { "type": "string", "format": "date" }, + "arena_score": { "type": "number", "minimum": 0, "maximum": 100 }, + "arena_score_ci95": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "prefixItems": [{ "type": "number", "minimum": 0, "maximum": 100 }, { "type": "number", "minimum": 0, "maximum": 100 }] + }, + "battle_count": { "type": "integer", "minimum": 0 }, + "deterministic_task_success": { "type": "number", "minimum": 0, "maximum": 1 }, + "failure_rate": { "type": "number", "minimum": 0, "maximum": 1 }, + "latency_ms_p50": { "type": "number", "minimum": 0 }, + "tokens": { "$ref": "#/$defs/tokens" }, + "estimated_cost_usd_per_run": { "type": "number", "minimum": 0 }, + "per_task_slices": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/taskSlice" } + } + } + }, + "tokens": { + "type": "object", + "additionalProperties": false, + "required": ["input", "output", "cache_read", "cache_write"], + "properties": { + "input": { "type": "integer", "minimum": 0 }, + "output": { "type": "integer", "minimum": 0 }, + "cache_read": { "type": "integer", "minimum": 0 }, + "cache_write": { "type": "integer", "minimum": 0 } + } + }, + "taskSlice": { + "type": "object", + "additionalProperties": false, + "required": ["task_id", "label", "success_rate", "battle_count"], + "properties": { + "task_id": { "type": "string", "minLength": 1 }, + "label": { "type": "string", "minLength": 1 }, + "success_rate": { "type": "number", "minimum": 0, "maximum": 1 }, + "battle_count": { "type": "integer", "minimum": 0 } + } + }, + "arenaBinding": { + "type": "object", + "additionalProperties": false, + "required": ["arena_id", "source_head_sha", "arena_contract_sha256", "parent_contract_sha256"], + "properties": { + "arena_id": { "const": "composer-archive-to-output-arena-v0" }, + "source_head_sha": { "const": "3dceb46215e999f4a11a564ea958f845c36af215" }, + "arena_contract_sha256": { "const": "8ceab1529c0bca8d74419222fe5b2556bceb1b83aaa9e3337b0509557ab1dab4" }, + "parent_contract_sha256": { "const": "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624" } + } + }, + "publication": { + "type": "object", + "additionalProperties": false, + "required": ["publishable", "state", "scope", "winner", "rank_count"], + "properties": { + "publishable": { "type": "boolean" }, + "state": { "enum": ["ranked", "tied", "insufficient_evidence"] }, + "scope": { "const": "independent_track_results_only" }, + "winner": { "type": ["string", "null"] }, + "rank_count": { "type": "integer", "minimum": 0 } + } + }, + "voteProvenance": { + "type": "object", + "additionalProperties": false, + "required": ["collection_mode", "crowdsourced", "real_human_votes"], + "properties": { + "collection_mode": { "enum": ["bounded_human_review", "exploratory_model_review", "synthetic_fixture", "no_eligible_votes"] }, + "crowdsourced": { "const": false }, + "real_human_votes": { "type": "integer", "minimum": 0 } + } + }, + "exploratoryCapability": { + "type": "object", + "additionalProperties": false, + "required": ["id", "title", "status", "summary", "evidence_scope", "limitation"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "status": { "const": "non_rankable" }, + "summary": { "type": "string", "minLength": 1 }, + "evidence_scope": { "type": "string", "minLength": 1 }, + "limitation": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/scripts/check-composer-arena.mjs b/scripts/check-composer-arena.mjs new file mode 100644 index 0000000..4e6c631 --- /dev/null +++ b/scripts/check-composer-arena.mjs @@ -0,0 +1,265 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import Ajv2020 from "ajv/dist/2020.js"; + +const root = resolve(import.meta.dirname, ".."); +const routeRoot = resolve(root, "src/app/bench/composer-arena"); +const artifactRoot = resolve(root, "public/bench/composer-arena/v0"); +const evidenceArtifactRoot = resolve(root, "public/bench/composer-arena/evidence/pilot/2026-07-19"); + +const release = JSON.parse(readFileSync(resolve(artifactRoot, "release.preview.json"), "utf8")); +const releaseSchema = JSON.parse(readFileSync(resolve(artifactRoot, "release.schema.json"), "utf8")); +const battleSchema = JSON.parse(readFileSync(resolve(artifactRoot, "battle.schema.json"), "utf8")); +const checksums = JSON.parse(readFileSync(resolve(artifactRoot, "checksums.json"), "utf8")); +const battles = readFileSync(resolve(artifactRoot, "battles.preview.jsonl"), "utf8") + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line)); + +const page = readFileSync(resolve(routeRoot, "page.tsx"), "utf8"); +const client = readFileSync(resolve(routeRoot, "arena-client.tsx"), "utf8"); +const data = readFileSync(resolve(routeRoot, "data.ts"), "utf8"); +const benchPage = readFileSync(resolve(root, "src/app/bench/page.tsx"), "utf8"); +const methodology = readFileSync(resolve(artifactRoot, "methodology.md"), "utf8"); +const handoff = readFileSync(resolve(artifactRoot, "handoff.md"), "utf8"); + +const evidence = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "seven-family-results.json"), "utf8")); +const evidenceSchema = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "seven-family-results.schema.json"), "utf8")); +const evidenceChecksums = JSON.parse(readFileSync(resolve(evidenceArtifactRoot, "checksums.json"), "utf8")); +const evidencePage = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/page.tsx"), "utf8"); +const evidenceData = readFileSync(resolve(routeRoot, "evidence/pilot/2026-07-19/data.ts"), "utf8"); + +function requireCondition(condition, message) { + if (!condition) throw new Error(message); +} + +function validateSchema(schemaDocument, value, label) { + const ajv = new Ajv2020({ allErrors: true, strict: false, validateFormats: false }); + const validate = ajv.compile(schemaDocument); + requireCondition(validate(value), `${label} schema validation failed: ${ajv.errorsText(validate.errors)}`); +} + +function sha256(base, filename) { + return createHash("sha256").update(readFileSync(resolve(base, filename))).digest("hex"); +} + +function requireChecksumSet(base, checksumDocument, label) { + for (const [filename, expected] of Object.entries(checksumDocument.files)) { + requireCondition(/^[0-9a-f]{64}$/.test(expected), `invalid ${label} checksum for ${filename}`); + requireCondition(existsSync(resolve(base, filename)), `missing checksummed ${label} artifact: ${filename}`); + requireCondition(sha256(base, filename) === expected, `${label} checksum mismatch for ${filename}`); + } +} + +const requiredArtifacts = [ + "release.schema.json", + "battle.schema.json", + "release.preview.json", + "battles.preview.jsonl", + "checksums.json", + "methodology.md", + "handoff.md", +]; + +for (const artifact of requiredArtifacts) { + requireCondition(existsSync(resolve(artifactRoot, artifact)), `missing Composer Arena artifact: ${artifact}`); + requireCondition(page.includes(artifact), `Composer Arena page does not link ${artifact}`); +} +requireCondition(page.includes('const ARTIFACT_ROOT = "/bench/composer-arena/v0"'), "Composer Arena artifact root drifted"); +requireChecksumSet(artifactRoot, checksums, "Composer Arena"); +requireChecksumSet(evidenceArtifactRoot, evidenceChecksums, "seven-family evidence"); + +requireCondition(releaseSchema.$id === "https://agentic.video/bench/composer-arena/v0/release.schema.json", "release schema ID drifted"); +requireCondition(releaseSchema.additionalProperties === false, "release schema must fail closed on unknown top-level fields"); +validateSchema(releaseSchema, release, "preview release"); +for (const battle of battles) validateSchema(battleSchema, battle, `battle ${battle.battle_id}`); + +requireCondition(release.release_mode === "synthetic_non_rankable_preview", "fixtures must remain a synthetic preview"); +requireCondition(release.data_class === "synthetic", "preview data class must remain synthetic"); +requireCondition(release.official_ranking_enabled === false, "official ranking must remain disabled"); +requireCondition(release.official_vote_backend === "not_configured", "preview must not imply a governed vote backend"); +requireCondition(release.display_label === "non-rankable preview", "preview label drifted"); +requireCondition(release.publication.publishable === false, "preview must not be publishable"); +requireCondition(release.publication.state === "insufficient_evidence", "preview publication state must fail closed"); +requireCondition(release.publication.winner === null && release.publication.rank_count === 0, "preview must have no winner or rank"); +requireCondition(release.vote_provenance.real_human_votes === 0, "preview must have zero real human votes"); +requireCondition(release.dataset.repo_id === "HuggingFaceFV/finevideo", "FineVideo repo drifted"); +requireCondition(release.dataset.revision === "84c74091e1c6ee7a5dffabfafb5c9033e4718883", "FineVideo revision drifted"); +requireCondition(release.dataset.license_claim === "CC-BY", "FineVideo licence claim drifted"); +requireCondition(release.dataset.finevideo_release_sha256 === "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", "FineVideo release hash drifted"); +requireCondition(release.dataset.split_policy.source_family_disjoint === true, "source families must remain disjoint"); +requireCondition(release.dataset.split_policy.hidden_test_published === false, "hidden test must remain unpublished"); + +const trackIds = release.tracks.map((track) => track.id).sort(); +requireCondition( + JSON.stringify(trackIds) === JSON.stringify(["controlled-agent", "end-to-end-system"]), + "release must contain exactly two separate Arena tracks", +); +const canonicalTrackIds = release.tracks.map((track) => track.canonical_track_id).sort(); +requireCondition( + JSON.stringify(canonicalTrackIds) === JSON.stringify(["controlled-semantic-text-evidence-v0", "end-to-end-visual-orchestration-v0"]), + "canonical Arena track IDs drifted", +); +for (const track of release.tracks) { + requireCondition(track.rows.length === 0, `${track.id} must publish zero standing rows`); +} +requireCondition( + release.exploratory_capabilities.every((capability) => capability.status === "non_rankable"), + "exploratory capabilities must remain non-rankable", +); + +requireCondition(battles.length === 2, "the generic demo must contain exactly two replay fixtures"); +requireCondition(new Set(battles.map((battle) => battle.track)).size === 2, "demo fixtures must cover both tracks"); +for (const battle of battles) { + requireCondition(battle.source.rankable === false, `${battle.battle_id} must remain non-rankable`); + requireCondition(battle.left.evidence.length > 0 && battle.right.evidence.length > 0, `${battle.battle_id} lacks evidence links`); + for (const candidate of [battle.left, battle.right]) { + requireCondition(candidate.model.startsWith("Synthetic demo "), `${battle.battle_id} leaks a non-generic candidate identity`); + requireCondition(candidate.provider === "Not applicable", `${battle.battle_id} leaks a provider identity`); + requireCondition(candidate.version.startsWith("fixture-"), `${battle.battle_id} leaks a route version`); + } +} + +const serializedPreview = JSON.stringify({ release, battles }).toLowerCase(); +for (const forbidden of [ + "gpt-5.6", + "grok 4.5", + "kimi k3", + "sol-medium-v1", + "grok-4.5-medium-v1", + "kimi-k3-high-v1", +]) { + requireCondition(!serializedPreview.includes(forbidden), `public preview contains a real-model fixture identity: ${forbidden}`); +} + +for (const control of ["Swap A/B order", "A is better", "B is better", "Tie", "Both bad", "Abstain"]) { + requireCondition(client.includes(control), `local demo is missing the ${control} control`); +} +requireCondition(client.includes("revealed={judgment !== null}"), "demo identity must reveal only after judgment"); +requireCondition(client.includes("No request is sent"), "local vote exclusion must be visible"); +requireCondition(data.includes("release.preview.json") && data.includes("battles.preview.jsonl"), "route must be artifact-driven"); + +requireCondition(methodology.includes("seven-family task-success"), "methodology is missing seven-family evidence"); +requireCondition(methodology.includes("source families must remain disjoint"), "methodology is missing the source-family split"); +requireCondition(methodology.includes("CC-BY"), "methodology is missing FineVideo attribution scope"); +requireCondition(methodology.includes("Abstain"), "methodology is missing the abstain path"); +requireCondition(methodology.includes("identity mapping remain private"), "methodology is missing the sealed-identity boundary"); +requireCondition(handoff.includes("battle.schema.json"), "handoff must bind JSONL rows to the battle schema"); +requireCondition(handoff.includes("governed vote backend"), "handoff is missing official vote governance"); +requireCondition(handoff.includes("SEA Broadcast ASR remains a separate benchmark suite"), "handoff must preserve the SEA-ASR boundary"); +requireCondition(handoff.includes("abstain"), "handoff is missing the abstain path"); +requireCondition(handoff.includes("outside `public/`"), "handoff is missing the private blind-packet boundary"); +requireCondition(benchPage.includes("Separate benchmark suite"), "SEA-ASR page must label Composer as a separate suite"); +requireCondition(benchPage.includes("does not change SEA Broadcast ASR"), "SEA-ASR page must state that Composer cannot change its result"); + +requireCondition(evidenceSchema.additionalProperties === false, "seven-family evidence schema must fail closed"); +validateSchema(evidenceSchema, evidence, "seven-family evidence"); +requireCondition(evidence.evidence_id === "composer-arena-seven-family-task-success-2026-07-19", "evidence ID drifted"); +requireCondition(evidence.publication.publishable === false, "seven-family evidence must not be publishable"); +requireCondition(evidence.publication.state === "insufficient_evidence", "seven-family evidence must remain insufficient"); +requireCondition(evidence.publication.leaderboard === false, "seven-family evidence must not be a leaderboard"); +requireCondition(evidence.publication.winner === null && evidence.publication.rank_count === 0, "seven-family evidence must have zero ranks and no winner"); +requireCondition(evidence.publication.human_preference_claims === false, "seven-family evidence must make no human preference claim"); +requireCondition( + evidence.vote_provenance.blind_battles === 0 && + evidence.vote_provenance.real_human_votes === 0 && + evidence.vote_provenance.resolved_battle_groups === 0, + "seven-family evidence must publish zero battles and votes", +); +requireCondition( + evidence.cohort.eligible_cells === 7 && + evidence.cohort.distinct_source_families === 7 && + evidence.cohort.favorable_selection === false && + evidence.cohort.source_family_disjoint === true, + "seven-family cohort gate drifted", +); + +const expectedHashes = { + staging_manifest_sha256: "0b395b961b0de0bb6a5c54ff80b13582d23f8e3d2e729aa7e950b1df8536d2b8", + contract_sha256: "33ca2f1b72ee9def3bd039eda69d9903ec595d860936a0353fdb045576215624", + release_sha256: "d37cd450934c49721cb75de14353e070759c8220f1ed87567799473e8ab88926", + case_pack_sha256: "74356b45867f492688e25a10c6e13a6fa2848f41ce00aa025d83adb7639ce203", +}; +for (const [key, expected] of Object.entries(expectedHashes)) { + requireCondition(evidence.frozen_evidence[key] === expected, `seven-family ${key} drifted`); +} + +const lanes = Object.fromEntries(evidence.task_success.lanes.map((lane) => [lane.lane_id, lane])); +const expectedLanes = { + "mechanical-control": { + classification: "control", + identity_state: "public_control", + structural_passes: 6, + structural_failures: 1, + complete_outputs_accepted: 5, + complete_output_failures: 2, + model_calls: 0, + source_summary_sha256: "3b856b8f1947b30da495b3a2fce20074270777d1f09b67b574dc318133291c57", + }, + "evaluated-lane-a": { + classification: "evaluated_system", + identity_state: "sealed_pending_founder_vote", + structural_passes: 2, + structural_failures: 5, + complete_outputs_accepted: 2, + complete_output_failures: 5, + model_calls: 7, + source_summary_sha256: "7505dd209740beb4c5696d778982de105996a01f53043ab550ce5888ae8efa1a", + }, + "evaluated-lane-b": { + classification: "evaluated_system", + identity_state: "sealed_pending_founder_vote", + structural_passes: 4, + structural_failures: 3, + complete_outputs_accepted: 4, + complete_output_failures: 3, + model_calls: 7, + source_summary_sha256: "adde8ddfb024bc1b1993ddc0ad72c79bfc362099c49ad6045bde9e0b1e22f456", + }, +}; +requireCondition(Object.keys(lanes).length === 3, "seven-family evidence must contain exactly three lanes"); +for (const [laneId, expected] of Object.entries(expectedLanes)) { + const lane = lanes[laneId]; + requireCondition(lane?.attempted_cells === 7, `${laneId} must contain seven attempted cells`); + for (const [key, value] of Object.entries(expected)) { + requireCondition(lane[key] === value, `${laneId} ${key} drifted`); + } +} + +for (const flag of ["packet_public", "candidate_outputs_public", "identity_mapping_public", "founder_vote_recorded"]) { + requireCondition(evidence.blind_review[flag] === false, `blind-review flag ${flag} must remain false`); +} + +const serializedEvidence = JSON.stringify(evidence).toLowerCase(); +for (const forbidden of [ + "/users/", + "/tmp/", + "http://", + "https://", + "s3://", + "signed_url", + "credential", + "password", + "secret", + "hidden_test", + "finevideo-f567", + "run_id", + "output_sha256", + "candidate_id", + "model_id", + "provider", +]) { + requireCondition(!serializedEvidence.includes(forbidden), `seven-family evidence contains forbidden public material: ${forbidden}`); +} + +requireCondition(evidencePage.includes("not a leaderboard"), "evidence route must visibly reject leaderboard framing"); +requireCondition(evidencePage.includes("Winner: none"), "evidence route must visibly reject a winner claim"); +for (const label of ["Source families", "Blind battles", "Published ranks", "Human votes"]) { + requireCondition(evidencePage.includes(label), `evidence route is missing the ${label} count`); +} +requireCondition(evidencePage.includes("no candidate media"), "evidence route must disclose the private packet boundary"); +requireCondition(evidenceData.includes("seven-family-results.json"), "evidence route must be artifact-driven"); + +console.log("Composer Arena route contract check passed"); diff --git a/scripts/next-font-google-mock.cjs b/scripts/next-font-google-mock.cjs new file mode 100644 index 0000000..8a4ea99 --- /dev/null +++ b/scripts/next-font-google-mock.cjs @@ -0,0 +1,11 @@ +module.exports = { + "https://fonts.googleapis.com/css2?family=Chivo+Mono:wght@100..900&display=swap": ` + @font-face { + font-family: 'Chivo Mono'; + font-style: normal; + font-weight: 100 900; + font-display: swap; + src: url(mock-chivo-mono.woff2) format('woff2'); + } + `, +}; diff --git a/src/app/bench/composer-arena/arena-client.tsx b/src/app/bench/composer-arena/arena-client.tsx new file mode 100644 index 0000000..b197520 --- /dev/null +++ b/src/app/bench/composer-arena/arena-client.tsx @@ -0,0 +1,381 @@ +"use client"; + +import { useMemo, useState } from "react"; + +import type { + ArenaBattle, + ArenaRelease, + ArenaTrackId, + BattleCandidate, +} from "./types"; + +type Judgment = "left" | "right" | "tie" | "both_bad" | "abstain"; + +const percent = new Intl.NumberFormat("en-US", { + style: "percent", + maximumFractionDigits: 1, +}); + +const integer = new Intl.NumberFormat("en-US"); + +function StandingBadge({ standing }: { standing: "ranked" | "tied" | "insufficient_evidence" }) { + if (standing === "insufficient_evidence") { + return ( + + insufficient evidence + + ); + } + + if (standing === "tied") { + return ( + + tied example + + ); + } + + return ( + + ranked + + ); +} + +function TrackTable({ release, trackId }: { release: ArenaRelease; trackId: ArenaTrackId }) { + const track = release.tracks.find((candidate) => candidate.id === trackId); + + if (!track) return null; + + if (track.rows.length === 0) { + return ( +
+
+

{track.title}

+

{track.description}

+ {track.canonical_track_id} +
+
+ No public standings are available. Task-success evidence stays separate from preference ranking. +
+
+ ); + } + + return ( +
+
+

{track.title}

+

{track.description}

+ {track.canonical_track_id} +
+ +
+ + + + + {[ + "Standing", + "Model / route", + "Arena score (95% CI)", + "Battles", + "Task success", + "Failure rate", + "p50 latency", + "Tokens", + "Est. cost / run", + "Per-task slices", + ].map((heading) => ( + + ))} + + + + {track.rows.map((row) => ( + + + + + + + + + + + + + ))} + +
{track.title} synthetic preview standings
+ {heading} +
+
{row.display_rank}
+
+ 		+
{row.model}
+
+ {row.provider}
{row.version}
{row.evaluation_date} +
+ 		+
{integer.format(row.arena_score)}
+
+ {integer.format(row.arena_score_ci95[0])}–{integer.format(row.arena_score_ci95[1])} +
+ 		{integer.format(row.battle_count)}{percent.format(row.deterministic_task_success)}{percent.format(row.failure_rate)}{(row.latency_ms_p50 / 1000).toFixed(1)}s + {integer.format(row.tokens.input)} in
+ {integer.format(row.tokens.output)} out
+ {integer.format(row.tokens.cache_read)} cache read + 		${row.estimated_cost_usd_per_run.toFixed(2)} +
    + {row.per_task_slices.map((slice) => ( +
  • +
    + {slice.label} + {percent.format(slice.success_rate)} +
    +
    • + ))} +
+
+
+
+ ); +} + +function CandidateCard({ + label, + candidate, + revealed, +}: { + label: "A" | "B"; + candidate: BattleCandidate; + revealed: boolean; +}) { + return ( +
+
+ + + {revealed ? candidate.model : "identity hidden"} + +
+

Output {label}

+

{candidate.output_summary}

+ + {revealed && ( +
+
+
+
Provider
+
{candidate.provider}
+
+
+
Version
+
{candidate.version}
+
+
+
+ {candidate.evidence.map((link) => ( + + {link.label} + + ))} +
+
+ )} +
+ ); +} + +function BattleReplay({ battles }: { battles: ArenaBattle[] }) { + const [battleIndex, setBattleIndex] = useState(0); + const [swapped, setSwapped] = useState(false); + const [judgment, setJudgment] = useState(null); + const battle = battles[battleIndex]; + + const [candidateA, candidateB] = useMemo( + () => (swapped ? [battle.right, battle.left] : [battle.left, battle.right]), + [battle, swapped], + ); + + const chooseBattle = (index: number) => { + setBattleIndex(index); + setSwapped(false); + setJudgment(null); + }; + + const swap = () => { + setSwapped((value) => !value); + setJudgment(null); + }; + + return ( +
+
+
+ + +
+ +
+ +
+
+ {battle.track}{battle.task_slice} +
+

{battle.prompt}

+

+ Synthetic source: {battle.source.fixture_id} · {battle.source.source_family_id} · non-rankable +

+
+ +
+ + +
+ +
+ Record a local demo judgment +

+ This selection stays in this page only. No request is sent and official standings never change. +

+
+ {([ + ["left", "A is better"], + ["tie", "Tie"], + ["right", "B is better"], + ["both_bad", "Both bad"], + ["abstain", "Abstain"], + ] as const).map(([value, label]) => ( + + ))} +
+
+ +
+ {judgment ? ( +

+ Identity revealed. Demo judgment recorded locally as {judgment.replace("_", " ")}; excluded from official ranking. +

+ ) : ( +

Choose only after comparing the anonymous outputs.

+ )} +
+
+ ); +} + +export default function ArenaClient({ release, battles }: { release: ArenaRelease; battles: ArenaBattle[] }) { + const [trackId, setTrackId] = useState("controlled-agent"); + + return ( + <> +
+
+
+
+

Standings gate

+

Separate views; no forced ranks

+
+
+ {release.tracks.map((track) => ( + + ))} +
+
+ +
+ No synthetic or measured score rows are published. A real standing requires at least {release.evaluation_policy.minimum_resolved_battles_per_pair_overall} resolved battles per pair plus the full frozen gates. +
+ + +
+
+ +
+
+

Exploratory capabilities

+

Observed separately; never converted into rank

+
+ {release.exploratory_capabilities.map((capability) => ( +
+ + non-rankable + +

{capability.title}

+

{capability.summary}

+

+ Evidence: {capability.evidence_scope} +

+

+ Limit: {capability.limitation} +

+
+ ))} +
+
+
+ +
+
+

Synthetic A/B interaction demo

+

Practice the judgment flow; no real packet is here

+

+ Swap order to test position sensitivity, choose a winner, tie, both-bad, or abstain, then reveal generic demo identities. The real blind packet and model mapping remain private. +

+
+
+
+ + ); +} diff --git a/src/app/bench/composer-arena/data.ts b/src/app/bench/composer-arena/data.ts new file mode 100644 index 0000000..d06d00e --- /dev/null +++ b/src/app/bench/composer-arena/data.ts @@ -0,0 +1,35 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +import type { ArenaBattle, ArenaChecksums, ArenaRelease } from "./types"; + +const artifactRoot = path.join( + process.cwd(), + "public", + "bench", + "composer-arena", + "v0", +); + +async function readJson(filename: string): Promise { + return JSON.parse(await readFile(path.join(artifactRoot, filename), "utf8")) as T; +} + +async function readJsonLines(filename: string): Promise { + const raw = await readFile(path.join(artifactRoot, filename), "utf8"); + return raw + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => JSON.parse(line) as T); +} + +export async function loadComposerArena() { + const [release, battles, checksums] = await Promise.all([ + readJson("release.preview.json"), + readJsonLines("battles.preview.jsonl"), + readJson("checksums.json"), + ]); + + return { release, battles, checksums }; +} diff --git a/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts new file mode 100644 index 0000000..5971aec --- /dev/null +++ b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/data.ts @@ -0,0 +1,88 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +export type TaskSuccessLane = { + lane_id: "mechanical-control" | "evaluated-lane-a" | "evaluated-lane-b"; + classification: "control" | "evaluated_system"; + identity_state: "public_control" | "sealed_pending_founder_vote"; + attempted_cells: 7; + structural_passes: number; + structural_failures: number; + complete_outputs_accepted: number; + complete_output_failures: number; + model_calls: number; + source_summary_sha256: string; +}; + +export type SevenFamilyEvidence = { + evidence_id: string; + evidence_date: string; + status: string; + publication: { + publishable: false; + state: "insufficient_evidence"; + leaderboard: false; + winner: null; + rank_count: 0; + human_preference_claims: false; + }; + vote_provenance: { + blind_battles: 0; + real_human_votes: 0; + resolved_battle_groups: 0; + }; + frozen_evidence: { + dataset_repo_id: string; + dataset_revision: string; + staging_manifest_sha256: string; + contract_sha256: string; + release_sha256: string; + case_pack_sha256: string; + }; + cohort: { + eligible_cells: 7; + distinct_source_families: 7; + favorable_selection: false; + source_family_disjoint: true; + }; + task_success: { + separation_statement: string; + lanes: TaskSuccessLane[]; + }; + blind_review: { + packet_public: false; + candidate_outputs_public: false; + identity_mapping_public: false; + founder_vote_recorded: false; + next_gate: string; + }; + limitations: string[]; +}; + +type Checksums = { + algorithm: "sha256"; + files: Record; +}; + +const artifactRoot = path.join( + process.cwd(), + "public", + "bench", + "composer-arena", + "evidence", + "pilot", + "2026-07-19", +); + +export async function loadSevenFamilyEvidence() { + const [evidence, checksums] = await Promise.all([ + readFile(path.join(artifactRoot, "seven-family-results.json"), "utf8").then( + (raw) => JSON.parse(raw) as SevenFamilyEvidence, + ), + readFile(path.join(artifactRoot, "checksums.json"), "utf8").then( + (raw) => JSON.parse(raw) as Checksums, + ), + ]); + + return { evidence, checksums }; +} diff --git a/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx new file mode 100644 index 0000000..e63650b --- /dev/null +++ b/src/app/bench/composer-arena/evidence/pilot/2026-07-19/page.tsx @@ -0,0 +1,191 @@ +import type { Metadata } from "next"; +import Link from "next/link"; + +import { loadSevenFamilyEvidence } from "./data"; + +const ARTIFACT_ROOT = "/bench/composer-arena/evidence/pilot/2026-07-19"; + +export const metadata: Metadata = { + title: "Composer Arena seven-family task-success evidence — July 19, 2026", + description: + "Sanitized seven-family deterministic task-success evidence with sealed model identities, zero blind battles, zero ranks, and no winner.", + alternates: { canonical: "/bench/composer-arena/evidence/pilot/2026-07-19" }, +}; + +function HashValue({ children }: { children: string }) { + return {children}; +} + +export default async function SevenFamilyEvidencePage() { + const { evidence, checksums } = await loadSevenFamilyEvidence(); + + return ( +
+
+
+
+ av + + Composer Arena + +
+ + Evidence / task success · 2026-07-19 + +
+
+ +
+
+ + not a leaderboard + +

+ Composer Archive-to-Output Arena v0 +

+

+ Seven-family task-success evidence +

+

+ A sanitized aggregate over seven eligible cells from seven distinct source families. It reports deterministic system outcomes only; it cannot establish editorial preference or an Arena winner. +

+ +
+ Winner: none. Publication state: insufficient evidence. Evaluated identities and the real blind packet remain private until the founder records a vote. +
+ +
+ {[ + ["Source families", evidence.cohort.distinct_source_families], + ["Blind battles", evidence.vote_provenance.blind_battles], + ["Published ranks", evidence.publication.rank_count], + ["Human votes", evidence.vote_provenance.real_human_votes], + ].map(([term, value]) => ( +
+
{term}
+
{value}
+
+ ))} +
+
+
+ +
+
+

Deterministic task success

+

Three lanes; no preference ordering

+

+ {evidence.task_success.separation_statement} +

+ +
+ + + + + {[ + "Lane", + "Identity", + "Attempted", + "Structural", + "Accepted outputs", + "Model calls", + "Source summary SHA-256", + ].map((heading) => ( + + ))} + + + + {evidence.task_success.lanes.map((lane) => ( + + + + + + + + + + ))} + +
Seven-family deterministic task-success lanes
{heading}
{lane.lane_id}{lane.identity_state.replaceAll("_", " ")}{lane.attempted_cells}{lane.structural_passes} pass / {lane.structural_failures} fail{lane.complete_outputs_accepted} / {lane.attempted_cells}{lane.model_calls}{lane.source_summary_sha256}
+
+
+
+ +
+
+

Blind-review boundary

+

Commit now; reveal after the vote

+
+ {[ + ["Packet public", evidence.blind_review.packet_public ? "Yes" : "No"], + ["Candidate outputs public", evidence.blind_review.candidate_outputs_public ? "Yes" : "No"], + ["Identity mapping public", evidence.blind_review.identity_mapping_public ? "Yes" : "No"], + ].map(([label, value]) => ( +
+

{label}

+

{value}

+
+ ))} +
+

+ Next gate: {evidence.blind_review.next_gate.replaceAll("_", " ")}. This page contains no candidate media, per-cell identifiers, output hashes, run IDs, prompts, private paths, or model mapping. +

+
+
+ +
+
+

Frozen provenance

+

Exact shared-input and aggregate hashes

+
+ {[ + ["Staging manifest", evidence.frozen_evidence.staging_manifest_sha256], + ["System-eval contract", evidence.frozen_evidence.contract_sha256], + ["FineVideo release", evidence.frozen_evidence.release_sha256], + ["Case pack", evidence.frozen_evidence.case_pack_sha256], + ].map(([label, value]) => ( +
+

{label}

+ {value} +
+ ))} +
+ +
+ {[ + ["Evidence JSON", `${ARTIFACT_ROOT}/seven-family-results.json`], + ["Evidence schema", `${ARTIFACT_ROOT}/seven-family-results.schema.json`], + ["Checksums", `${ARTIFACT_ROOT}/checksums.json`], + ["Public README", `${ARTIFACT_ROOT}/README.md`], + ].map(([label, href]) => ( + + {label} + + ))} +
+

+ Evidence artifact SHA-256: {checksums.files["seven-family-results.json"]} +

+
+
+ +
+
+

Limits

+

What this evidence does not prove

+
    + {evidence.limitations.map((limitation, index) => ( +
  1. + {String(index + 1).padStart(2, "0")} + {limitation} +
    2. + ))} +
+
+
+
+ ); +} diff --git a/src/app/bench/composer-arena/page.tsx b/src/app/bench/composer-arena/page.tsx new file mode 100644 index 0000000..4a81861 --- /dev/null +++ b/src/app/bench/composer-arena/page.tsx @@ -0,0 +1,257 @@ +import type { Metadata } from "next"; +import Link from "next/link"; + +import ArenaClient from "./arena-client"; +import { loadComposerArena } from "./data"; + +const ARTIFACT_ROOT = "/bench/composer-arena/v0"; + +export const metadata: Metadata = { + title: "Composer Archive-to-Output Arena v0 — Agentic Video Benchmarks", + description: + "Non-rankable Composer Archive-to-Output Arena v0 methodology, seven-family task-success evidence, and synthetic interaction demo.", + alternates: { canonical: "/bench/composer-arena" }, + openGraph: { + title: "Composer Archive-to-Output Arena v0", + description: "Seven-family task-success evidence with zero ranks and a separate synthetic interaction demo.", + type: "website", + url: "https://agentic.video/bench/composer-arena", + }, +}; + +function SectionHeading({ eyebrow, children }: { eyebrow: string; children: React.ReactNode }) { + return ( +
+

{eyebrow}

+

{children}

+
+ ); +} + +function HashValue({ children }: { children: string }) { + return {children}; +} + +export default async function ComposerArenaPage() { + const { release, battles, checksums } = await loadComposerArena(); + + const artifactLinks = [ + ["Release fixture (JSON)", `${ARTIFACT_ROOT}/release.preview.json`], + ["Battle replay fixtures (JSONL)", `${ARTIFACT_ROOT}/battles.preview.jsonl`], + ["Release schema (JSON Schema)", `${ARTIFACT_ROOT}/release.schema.json`], + ["Battle schema (JSON Schema)", `${ARTIFACT_ROOT}/battle.schema.json`], + ["Checksums", `${ARTIFACT_ROOT}/checksums.json`], + ["Methodology", `${ARTIFACT_ROOT}/methodology.md`], + ["Real-data handoff", `${ARTIFACT_ROOT}/handoff.md`], + ] as const; + + return ( +
+
+
+
+ av + + Benchmarks + +
+ + Composer Arena by Pixel ML + +
+
+ +
+
+
+ {release.display_label} +
+

+ Agentic Video Benchmarks by Pixel ML +

+

+ Composer Archive-to-Output Arena v0 +

+

+ A publication-candidate methodology for evidence-grounded editing agents and complete Composer systems. The dated evidence page now reports the terminal seven-family task-success aggregate; no public standings exist. +

+ +
+ Official ranking is disabled. There are no public score rows, the replay is a generic synthetic interaction demo, local judgments are excluded, and evaluated identities stay sealed until the founder votes. +
+ +
+ {[ + ["Release", release.release_id], + ["Mode", release.release_mode], + ["Generated", release.generated_at], + ["Vote backend", release.official_vote_backend], + ].map(([term, value]) => ( +
+
{term}
+
{value}
+
+ ))} +
+
+
+ +
+
+
+
+ + evidence / pilot · not a leaderboard + +

Seven-family task-success evidence is available

+

+ The frozen cohort covers seven eligible cells from seven distinct source families. It reports one public mechanical control and two identity-sealed evaluated lanes, with zero blind battles, votes, ranks, or winner. +

+
+ + Open dated evidence + +
+
+
+ + + +
+
+ Deterministic evidence first; preference stays bounded +
+ {[ + [ + "Standing policy", + `${release.evaluation_policy.tie_rule} ${release.evaluation_policy.insufficient_evidence_rule}`, + ], + [ + "Vote boundary", + release.evaluation_policy.local_vote_policy, + ], + [ + "Contamination gate", + release.evaluation_policy.contamination_limit, + ], + ].map(([title, body]) => ( +
+

{title}

+

{body}

+
+ ))} +
+
+

Privacy and publication boundary

+

+ {release.evaluation_policy.privacy_limit} +

+
+
+
+ +
+
+
+ Immutable revision and bounded CC-BY claim +

+ {release.dataset.license_scope} +

+

+ Attribution: {release.dataset.attribution} +

+ + View the immutable FineVideo dataset revision + +
+ +
+ Public evidence is not hidden-test evidence +
+
+
Audited public-eval families
+
{release.dataset.public_eval_source_family_count}
+
+
+
Hidden test published
+
No
+
+
+
    +
  • Public eval: {release.dataset.split_policy.public_eval}
    • +
  • Private test: {release.dataset.split_policy.private_test}
    • +
  • Disjoint: the same source family cannot cross the split.
    • +
+
+
+
+ +
+
+ Contract, source release, and public artifact hashes +
+
+

System-eval contract SHA-256

+ {release.arena_binding.arena_contract_sha256} +
+
+

Parent system-eval contract SHA-256

+ {release.arena_binding.parent_contract_sha256} +
+
+

FineVideo release SHA-256

+ {release.dataset.finevideo_release_sha256} +
+
+

+ Public preview release SHA-256: {checksums.files["release.preview.json"]} · Arena source head: {release.arena_binding.source_head_sha} +

+

+ The command-room FineVideo selection and release marker pass for 24 cells. Seven-family deterministic task-success evidence is terminal, but blind human review and identity reveal are still pending, so no official ranking exists. +

+
+
+ +
+
+ What this preview does not prove +
    + {release.known_limitations.map((limitation, index) => ( +
  1. + {String(index + 1).padStart(2, "0")} + {limitation} +
    2. + ))} +
+
+
+ +
+
+ Downloadable, replaceable public artifacts +

+ The route reads the JSON and JSONL artifacts at build time. A governed real release replaces the files under the same schema and filenames, so the public UI does not require model-specific code changes. +

+
+ {artifactLinks.map(([label, href]) => ( + + {label} + + ))} +
+
+
+
+ ); +} diff --git a/src/app/bench/composer-arena/types.ts b/src/app/bench/composer-arena/types.ts new file mode 100644 index 0000000..1e9acfa --- /dev/null +++ b/src/app/bench/composer-arena/types.ts @@ -0,0 +1,148 @@ +export type ArenaTrackId = "controlled-agent" | "end-to-end-system"; + +export type ArenaStanding = "ranked" | "tied" | "insufficient_evidence"; + +export type TokenUsage = { + input: number; + output: number; + cache_read: number; + cache_write: number; +}; + +export type TaskSlice = { + task_id: string; + label: string; + success_rate: number; + battle_count: number; +}; + +export type ArenaRow = { + standing: ArenaStanding; + display_rank: string; + model: string; + provider: string; + version: string; + evaluation_date: string; + arena_score: number; + arena_score_ci95: [number, number]; + battle_count: number; + deterministic_task_success: number; + failure_rate: number; + latency_ms_p50: number; + tokens: TokenUsage; + estimated_cost_usd_per_run: number; + per_task_slices: TaskSlice[]; +}; + +export type ArenaTrack = { + id: ArenaTrackId; + canonical_track_id: "controlled-semantic-text-evidence-v0" | "end-to-end-visual-orchestration-v0"; + title: string; + description: string; + rows: ArenaRow[]; +}; + +export type ExploratoryCapability = { + id: string; + title: string; + status: "non_rankable"; + summary: string; + evidence_scope: string; + limitation: string; +}; + +export type ArenaRelease = { + schema_version: 1; + contract_version: "composer-arena-public-release-v0.1"; + release_id: string; + generated_at: string; + release_mode: "synthetic_non_rankable_preview" | "governed_rankable_release"; + data_class: "rankable" | "exploratory" | "synthetic"; + official_ranking_enabled: boolean; + official_vote_backend: "not_configured" | "governed_v1"; + display_label: string; + arena_binding: { + arena_id: "composer-archive-to-output-arena-v0"; + source_head_sha: string; + arena_contract_sha256: string; + parent_contract_sha256: string; + }; + publication: { + publishable: boolean; + state: "ranked" | "tied" | "insufficient_evidence"; + scope: "independent_track_results_only"; + winner: string | null; + rank_count: number; + }; + vote_provenance: { + collection_mode: "bounded_human_review" | "exploratory_model_review" | "synthetic_fixture" | "no_eligible_votes"; + crowdsourced: false; + real_human_votes: number; + }; + dataset: { + repo_id: string; + revision: string; + license_claim: string; + license_scope: string; + attribution: string; + finevideo_release_sha256: string; + selection_manifest_sha256: string; + public_eval_source_family_count: number; + split_policy: { + source_family_disjoint: true; + public_eval: string; + private_test: string; + hidden_test_published: false; + }; + }; + evaluation_policy: { + ranking_method: "Bradley-Terry logistic maximum likelihood"; + confidence_level: 0.95; + bootstrap_replicates: 2000; + minimum_resolved_battles_per_pair_overall: 4; + position_bias_decisive_presentations: 20; + tie_rule: string; + insufficient_evidence_rule: string; + local_vote_policy: string; + contamination_limit: string; + privacy_limit: string; + }; + tracks: ArenaTrack[]; + exploratory_capabilities: ExploratoryCapability[]; + known_limitations: string[]; +}; + +export type EvidenceLink = { + label: string; + href: string; +}; + +export type BattleCandidate = { + candidate_id: string; + model: string; + provider: string; + version: string; + output_title: string; + output_summary: string; + evidence: EvidenceLink[]; +}; + +export type ArenaBattle = { + battle_id: string; + track: ArenaTrackId; + task_slice: string; + prompt: string; + source: { + fixture_id: string; + source_family_id: string; + revision: string; + rankable: boolean; + }; + left: BattleCandidate; + right: BattleCandidate; +}; + +export type ArenaChecksums = { + algorithm: "sha256"; + files: Record; +}; diff --git a/src/app/bench/page.tsx b/src/app/bench/page.tsx index 9396288..a519c3f 100644 --- a/src/app/bench/page.tsx +++ b/src/app/bench/page.tsx @@ -128,6 +128,31 @@ export default function BenchPage() { +
+
+ Composer Archive-to-Output Arena v0 +
+
+
+ + non-rankable preview + +

Archive understanding to reviewable output

+

+ Composer Archive-to-Output uses a separate dataset, task contract, evidence page, and publication gate. Its seven-family task-success aggregate is non-rankable and does not change SEA Broadcast ASR. +

+
+ + Open Composer Arena + +
+
+
+
+
Rights and audio passed; human gold still stops execution