From 638e1ec4ca0e27ae424786c11590d1b5bd2a32a4 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:18:10 +0100 Subject: [PATCH 01/22] Retain Helix Nix results and native registry dispatches --- migration/evidence.json | 14 + .../1-reapi-fresh-benchmark.yml.json | 35 + .../1-reapi-fresh-benchmark.yml.json | 35 + .../completions/37296210599.json | 13 + .../dispatches/1-nix-fresh-benchmark.yml.json | 37 + .../rejected/1-nix-fresh-benchmark.yml.json | 35 + .../helix-nix/canary-screening-02/report.json | 752 ++++++++++++++++++ .../helix-nix/canary-screening-02/report.md | 29 + .../runs/1-boringcache-cold.json | 187 +++++ .../runs/1-boringcache-warm.json | 188 +++++ .../runs/1-cachix-cold.json | 145 ++++ .../runs/1-cachix-warm.json | 145 ++++ .../1-reapi-fresh-benchmark.yml.json | 36 + .../1-reapi-fresh-benchmark.yml.json | 35 + .../1-reapi-fresh-benchmark.yml.json | 35 + 15 files changed, 1721 insertions(+) create mode 100644 results/executorch-buck2/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/gogs-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/helix-nix/canary-screening-02/completions/37296210599.json create mode 100644 results/helix-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json create mode 100644 results/helix-nix/canary-screening-02/dispatches/rejected/1-nix-fresh-benchmark.yml.json create mode 100644 results/helix-nix/canary-screening-02/report.json create mode 100644 results/helix-nix/canary-screening-02/report.md create mode 100644 results/helix-nix/canary-screening-02/runs/1-boringcache-cold.json create mode 100644 results/helix-nix/canary-screening-02/runs/1-boringcache-warm.json create mode 100644 results/helix-nix/canary-screening-02/runs/1-cachix-cold.json create mode 100644 results/helix-nix/canary-screening-02/runs/1-cachix-warm.json create mode 100644 results/msgpack-sbt/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/opencut-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/stackstorm-pants/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json diff --git a/migration/evidence.json b/migration/evidence.json index c5e2aa92a..ac5786910 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -1273,6 +1273,20 @@ "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", "execution_qualification": "Cold and changed-source output checks passed; restore-only consumer, no rolling publication proof; storage unmeasured", "repository_retirement": "This run does not approve retirement of any original repository" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37296210599, + "verified_files": 17, + "gaps": [], + "sha256": "a888a59f9db1f52e986d17c2062d2ef363e69d5367861a735e65b61a99cc7ac1", + "bundle_bytes": 1840218, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-helix-nix-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-helix-nix-2026-10-05/benchmarks-37296210599.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" } ] } diff --git a/results/executorch-buck2/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json b/results/executorch-buck2/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..86823190c --- /dev/null +++ b/results/executorch-buck2/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:17:38Z", + "ref": "main", + "state": "requested", + "run_id": 37302007157, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157" +} diff --git a/results/gogs-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json b/results/gogs-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..975213412 --- /dev/null +++ b/results/gogs-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:17:32Z", + "ref": "main", + "state": "requested", + "run_id": 37301997879, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37301997879" +} diff --git a/results/helix-nix/canary-screening-02/completions/37296210599.json b/results/helix-nix/canary-screening-02/completions/37296210599.json new file mode 100644 index 000000000..db1263697 --- /dev/null +++ b/results/helix-nix/canary-screening-02/completions/37296210599.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296210599", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296210599", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a75d624e02f00f92884e89ebc0ed3201885a7ffa2e4cb898b238b26f5f10da9" +} diff --git a/results/helix-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json b/results/helix-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..0389de79d --- /dev/null +++ b/results/helix-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "canary-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T10:22:52Z", + "ref": "run-tool-family-cases", + "state": "requested", + "run_id": 37296210599, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296210599" +} diff --git a/results/helix-nix/canary-screening-02/dispatches/rejected/1-nix-fresh-benchmark.yml.json b/results/helix-nix/canary-screening-02/dispatches/rejected/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..30ec64a7e --- /dev/null +++ b/results/helix-nix/canary-screening-02/dispatches/rejected/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "canary-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T10:22:03Z", + "ref": "run-tool-family-cases", + "state": "planned" +} diff --git a/results/helix-nix/canary-screening-02/report.json b/results/helix-nix/canary-screening-02/report.json new file mode 100644 index 000000000..4b11b3813 --- /dev/null +++ b/results/helix-nix/canary-screening-02/report.json @@ -0,0 +1,752 @@ +{ + "schema_version": 1, + "plan_sha256": "c091853f2c86baae18c7f93abe5e4dc93476624cd5833cab107488beaaacbcb0", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296210599", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296210599", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a75d624e02f00f92884e89ebc0ed3201885a7ffa2e4cb898b238b26f5f10da9" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 296.34937598663436, + "min": 296.34937598663436, + "max": 296.34937598663436 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 5.810530557688935, + "min": 5.810530557688935, + "max": 5.810530557688935 + }, + "storage": { + "median": 46172357, + "min": 46172357, + "max": 46172357 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 360.6637888889659, + "min": 360.6637888889659, + "max": 360.6637888889659 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 4.730110110390228, + "min": 4.730110110390228, + "max": 4.730110110390228 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5654776096343994, + "build_seconds": 292.78389837699996, + "workflow_seconds": null, + "total_seconds": 296.34937598663436, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 296.34937598663436 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.180853843688965, + "build_seconds": 0.6296767139999702, + "workflow_seconds": null, + "total_seconds": 5.810530557688935, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 5.810530557688935 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46172357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "complete": true, + "measured_bytes": 46172357, + "unmeasured_tags": [], + "total_bytes": 46172357, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46172357, + "compressed_size": 46172357, + "size": 46172357 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173804", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.7561771869659424, + "build_seconds": 357.90761170199994, + "workflow_seconds": null, + "total_seconds": 360.6637888889659, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 360.6637888889659 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] + }, + { + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3965330123901367, + "build_seconds": 2.3335770980000916, + "workflow_seconds": null, + "total_seconds": 4.730110110390228, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 4.730110110390228 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173803", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] + } + ], + "exclusions": [] +} diff --git a/results/helix-nix/canary-screening-02/report.md b/results/helix-nix/canary-screening-02/report.md new file mode 100644 index 000000000..6e7f37b60 --- /dev/null +++ b/results/helix-nix/canary-screening-02/report.md @@ -0,0 +1,29 @@ +# helix-nix: canary-screening-02 + +Question: Does Nix restore the Helix package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer + +Observations: 4/4 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 1 | 296.34937598663436 | 296.34937598663436–296.34937598663436 | unmeasured | 0 | +| Warm build | BoringCache | 1 | 5.810530557688935 | 5.810530557688935–5.810530557688935 | 46172357 | 1 | +| Cold build | Cachix | 1 | 360.6637888889659 | 360.6637888889659–360.6637888889659 | unmeasured | 0 | +| Warm build | Cachix | 1 | 4.730110110390228 | 4.730110110390228–4.730110110390228 | unmeasured | 0 | + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 3.5654776096343994 | 292.78389837699996 | 296.34937598663436 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 5.180853843688965 | 0.6296767139999702 | 5.810530557688935 | 46172357 | boringcache-check | hit | [JSON](runs/1-boringcache-warm.json) | +| 1 | Cachix | Cold build | 2.7561771869659424 | 357.90761170199994 | 360.6637888889659 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | +| 1 | Cachix | Warm build | 2.3965330123901367 | 2.3335770980000916 | 4.730110110390228 | unmeasured | unmeasured | hit | [JSON](runs/1-cachix-warm.json) | + +[Full records and checks](report.json) diff --git a/results/helix-nix/canary-screening-02/runs/1-boringcache-cold.json b/results/helix-nix/canary-screening-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..8566342d7 --- /dev/null +++ b/results/helix-nix/canary-screening-02/runs/1-boringcache-cold.json @@ -0,0 +1,187 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3.5654776096343994, + "build_seconds": 292.78389837699996, + "workflow_seconds": null, + "total_seconds": 296.34937598663436, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 296.34937598663436 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173729", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] +} diff --git a/results/helix-nix/canary-screening-02/runs/1-boringcache-warm.json b/results/helix-nix/canary-screening-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..02b24f6d3 --- /dev/null +++ b/results/helix-nix/canary-screening-02/runs/1-boringcache-warm.json @@ -0,0 +1,188 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 5.180853843688965, + "build_seconds": 0.6296767139999702, + "workflow_seconds": null, + "total_seconds": 5.810530557688935, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 5.810530557688935 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 46172357, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "complete": true, + "measured_bytes": 46172357, + "unmeasured_tags": [], + "total_bytes": 46172357, + "observations": [ + { + "requested_tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "tag": "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 46172357, + "compressed_size": 46172357, + "size": 46172357 + } + ] + } + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "helix-nix-helix-nix-series-canary-screening-02-s1-r37296210599-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173804", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] +} diff --git a/results/helix-nix/canary-screening-02/runs/1-cachix-cold.json b/results/helix-nix/canary-screening-02/runs/1-cachix-cold.json new file mode 100644 index 000000000..495d5c966 --- /dev/null +++ b/results/helix-nix/canary-screening-02/runs/1-cachix-cold.json @@ -0,0 +1,145 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.7561771869659424, + "build_seconds": 357.90761170199994, + "workflow_seconds": null, + "total_seconds": 360.6637888889659, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 360.6637888889659 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173728", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] +} diff --git a/results/helix-nix/canary-screening-02/runs/1-cachix-warm.json b/results/helix-nix/canary-screening-02/runs/1-cachix-warm.json new file mode 100644 index 000000000..ab4da0c35 --- /dev/null +++ b/results/helix-nix/canary-screening-02/runs/1-cachix-warm.json @@ -0,0 +1,145 @@ +{ + "schema_version": 1, + "benchmark": "helix-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "helix-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "origin": { + "repository": "helix-editor/helix", + "branch": "master", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "helix-nix": "helix-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3965330123901367, + "build_seconds": 2.3335770980000916, + "workflow_seconds": null, + "total_seconds": 4.730110110390228, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 4.730110110390228 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "helix-editor/helix", + "sha": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296210599", + "run_attempt": "1", + "job": "warm", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173803", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296210599-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + ] +} diff --git a/results/msgpack-sbt/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json b/results/msgpack-sbt/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..66e0e8419 --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-03", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:17:39Z", + "ref": "main", + "state": "requested", + "run_id": 37302010045, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302010045" +} diff --git a/results/opencut-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json b/results/opencut-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..6c35afdfe --- /dev/null +++ b/results/opencut-moon/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:17:34Z", + "ref": "main", + "state": "requested", + "run_id": 37302001164, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164" +} diff --git a/results/stackstorm-pants/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json b/results/stackstorm-pants/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..1db494be9 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-02/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:17:36Z", + "ref": "main", + "state": "requested", + "run_id": 37302004103, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103" +} From 6aebc8f9f33a9e8de209e747b48c0a4f2fdda60a Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:20:34 +0100 Subject: [PATCH 02/22] Prepare the StackStorm test submodule --- .../payload/reapi-recipe.json | 7 +++ .../registry-screening-03/series.json | 48 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 results/stackstorm-pants/registry-screening-03/series.json diff --git a/cases/stackstorm-pants/payload/reapi-recipe.json b/cases/stackstorm-pants/payload/reapi-recipe.json index 616f4e0f5..500daa5e5 100644 --- a/cases/stackstorm-pants/payload/reapi-recipe.json +++ b/cases/stackstorm-pants/payload/reapi-recipe.json @@ -9,6 +9,13 @@ "pants-plugins/pack_metadata/target_types_test.py" ], "prepare": [ + [ + "git", + "submodule", + "update", + "--init", + "--recursive" + ], [ "pants", "--version" diff --git a/results/stackstorm-pants/registry-screening-03/series.json b/results/stackstorm-pants/registry-screening-03/series.json new file mode 100644 index 000000000..20d1ddb9f --- /dev/null +++ b/results/stackstorm-pants/registry-screening-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", + "created_at": "2026-10-05T11:20:34Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "72cc5c0680d703db46455d69fda311472e215b8423b3ae95b8276f4c8b040e4f", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "7cd6568e943d76372434e5ba184fd8204575671dc220d2c042b3e6cbd9200034" +} From 6d0b386d1fd5cd6f09d0694ffcd481a9b8ca8f0a Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:21:25 +0100 Subject: [PATCH 03/22] Use Moon 2.5 for the read-only OpenCut replay --- cases/opencut-moon/payload/mise.toml | 2 +- cases/opencut-moon/proposal.md | 6 ++- .../registry-screening-03/series.json | 48 +++++++++++++++++++ .../1-reapi-fresh-benchmark.yml.json | 35 ++++++++++++++ 4 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 results/opencut-moon/registry-screening-03/series.json create mode 100644 results/stackstorm-pants/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json diff --git a/cases/opencut-moon/payload/mise.toml b/cases/opencut-moon/payload/mise.toml index 711da1a32..4dccfd398 100644 --- a/cases/opencut-moon/payload/mise.toml +++ b/cases/opencut-moon/payload/mise.toml @@ -1,4 +1,4 @@ [tools] node = "24.13.0" bun = "1.3.11" -"npm:@moonrepo/cli" = "2.3.3" +"npm:@moonrepo/cli" = "2.5.6" diff --git a/cases/opencut-moon/proposal.md b/cases/opencut-moon/proposal.md index c997ae8e1..99215230b 100644 --- a/cases/opencut-moon/proposal.md +++ b/cases/opencut-moon/proposal.md @@ -2,8 +2,10 @@ Status: native REAPI correctness screen implemented; hosted qualification pending. -The shared Moon configuration enables `experiments.casOutputsCache`: the pinned -Moon 2.3.3 uploads remote outputs through that cache path. +The benchmark uses Moon 2.5.6 instead of upstream's 2.3.3. The older client rejects +a registry that advertises disabled action-cache updates, including a read-only +warm registry. Both providers use the same client version. The shared configuration +enables `experiments.casOutputsCache`. Source: [OpenCut-app/OpenCut at `e668010778568641babef2cc40be4703ae6916d6`](https://github.com/OpenCut-app/OpenCut/tree/e668010778568641babef2cc40be4703ae6916d6). diff --git a/results/opencut-moon/registry-screening-03/series.json b/results/opencut-moon/registry-screening-03/series.json new file mode 100644 index 000000000..daf127ca7 --- /dev/null +++ b/results/opencut-moon/registry-screening-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-03", + "created_at": "2026-10-05T11:21:25Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "d804908655d0ff3aae18750d5c32c8c4c74b07fc140fbd8d5a2cc8ac3feb6d87", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "228b7e3a051d81a1cb57a81e265507e7e6beef5c8ee0181fd54dc9e1f71b43ae" +} diff --git a/results/stackstorm-pants/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json b/results/stackstorm-pants/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..7d2fc4294 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-03", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:20:48Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302345686, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686" +} From 4ba17a6c2b55c5346a01d156445485a823718c45 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:21:40 +0100 Subject: [PATCH 04/22] Remove the outdated OpenCut toolchain statement --- cases/opencut-moon/proposal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cases/opencut-moon/proposal.md b/cases/opencut-moon/proposal.md index 99215230b..079ce9d8e 100644 --- a/cases/opencut-moon/proposal.md +++ b/cases/opencut-moon/proposal.md @@ -11,7 +11,7 @@ Source: [OpenCut-app/OpenCut at `e668010778568641babef2cc40be4703ae6916d6`](http ## Workload -The upstream web build is unchanged. The pinned revision has no lockfile; the payload adds a Bun 1.3.11 lockfile resolved on October 5, 2026. This is an explicit dependency-resolution change. Both providers use frozen installation against that file. The case retains upstream Moon 2.3.3. +The upstream web build is unchanged. The pinned revision has no lockfile; the payload adds a Bun 1.3.11 lockfile resolved on October 5, 2026. This is an explicit dependency-resolution change. Both providers use frozen installation against that file. Declared command: `moon run web:build`. From a15241e5d23058d6fa8890ab4e16fd15e319d417 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:21:54 +0100 Subject: [PATCH 05/22] Declare the OpenCut Moon 2.5 screen --- .../registry-screening-04/series.json | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 results/opencut-moon/registry-screening-04/series.json diff --git a/results/opencut-moon/registry-screening-04/series.json b/results/opencut-moon/registry-screening-04/series.json new file mode 100644 index 000000000..a458eb754 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-04", + "created_at": "2026-10-05T11:21:54Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "7016af095356764099bf0da79cec88f91affb18267ec18b5ebebdc310acdff72" +} From 829fa130f334a8af9df0c3312027921e2f2caaec Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:23:36 +0100 Subject: [PATCH 06/22] Install the pinned ExecuTorch code generator dependency --- .../payload/reapi-recipe.json | 8 ++++ cases/executorch-buck2/proposal.md | 5 ++ .../registry-screening-03/series.json | 48 +++++++++++++++++++ .../1-reapi-fresh-benchmark.yml.json | 35 ++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 results/executorch-buck2/registry-screening-03/series.json create mode 100644 results/opencut-moon/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json diff --git a/cases/executorch-buck2/payload/reapi-recipe.json b/cases/executorch-buck2/payload/reapi-recipe.json index 288ac9fac..8b2646b56 100644 --- a/cases/executorch-buck2/payload/reapi-recipe.json +++ b/cases/executorch-buck2/payload/reapi-recipe.json @@ -20,6 +20,14 @@ "pip", "install", "PyYAML==6.0.3" + ], + [ + "python3", + "-m", + "pip", + "install", + "--no-deps", + "https://download-r2.pytorch.org/whl/cpu/torch-2.14.0%2Bcpu-cp311-cp311-manylinux_2_28_x86_64.whl#sha256=673dbf5c9bbadfffab7a386b6dd7a0c219f1408a328b7b4e86d0ae551cdafa42" ] ], "outputs": [ diff --git a/cases/executorch-buck2/proposal.md b/cases/executorch-buck2/proposal.md index ca6e916e0..1d23862b2 100644 --- a/cases/executorch-buck2/proposal.md +++ b/cases/executorch-buck2/proposal.md @@ -6,6 +6,11 @@ Source: [pytorch/executorch at `5e21c13cc9e34fa2922d5708ba5962e3e365afdb`](https ## Workload +Preparation installs the pinned PyTorch 2.14.0 CPU wheel, verified by its published +SHA-256, for the upstream `torchgen_files` generator. It does not install CUDA +dependencies. + + The upstream portable executor runner target and Buck2 2025-05-06 pin are retained. The payload adds a local execution platform with remote caching enabled and remote execution disabled. Source submodules retain upstream pins. Verification checks executable ELF output and cold/warm hashes; it does not run an exported model. Declared command: `buck2 build //examples/portable/executor_runner:executor_runner --show-output`. diff --git a/results/executorch-buck2/registry-screening-03/series.json b/results/executorch-buck2/registry-screening-03/series.json new file mode 100644 index 000000000..158050cc5 --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "created_at": "2026-10-05T11:23:36Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "29364c9b6c1bdf023476a589a9b69a641cee3d1264058927f4e8d8ddd7592ed9", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "2ded7301f2b56ac93494c09c0bb722d6c01766e1528c760f6af15cbce49b25bf" +} diff --git a/results/opencut-moon/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json b/results/opencut-moon/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..4edbaae73 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:22:04Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302476504, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302476504" +} From 09efe2188e21ac30bde55b760b6d3f0b23d24ac5 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:24:02 +0100 Subject: [PATCH 07/22] Declare repeated Moon and sbt correctness checks --- .../1-reapi-fresh-benchmark.yml.json | 35 +++++++++++++ .../registry-qualification-01/series.json | 48 ++++++++++++++++++ .../registry-qualification-01/series.json | 49 +++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 results/executorch-buck2/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/gogs-moon/registry-qualification-01/series.json create mode 100644 results/msgpack-sbt/registry-qualification-01/series.json diff --git a/results/executorch-buck2/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json b/results/executorch-buck2/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..eb01ae847 --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-03", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:24:00Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302678892, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892" +} diff --git a/results/gogs-moon/registry-qualification-01/series.json b/results/gogs-moon/registry-qualification-01/series.json new file mode 100644 index 000000000..341d81f27 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "created_at": "2026-10-05T11:24:02Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "variant": null, + "workflow_inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "6f0af9bb2ddd8c15c7be1a0810617d8973841bcd8f8ff77dc61ef299041cd5c5" +} diff --git a/results/msgpack-sbt/registry-qualification-01/series.json b/results/msgpack-sbt/registry-qualification-01/series.json new file mode 100644 index 000000000..c6ad4db6d --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "created_at": "2026-10-05T11:24:02Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "variant": null, + "workflow_inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "796afbd8d1867db7442db953d38cb7e35d757fb18ef0d8825213983de643110e" +} From d63d3593c150f7471c7419a5bac6827c2979c3ef Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:25:33 +0100 Subject: [PATCH 08/22] Use the plugin test resolve for StackStorm screening --- .../payload/reapi-recipe.json | 3 ++ cases/stackstorm-pants/proposal.md | 6 ++- .../registry-screening-04/series.json | 48 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 results/stackstorm-pants/registry-screening-04/series.json diff --git a/cases/stackstorm-pants/payload/reapi-recipe.json b/cases/stackstorm-pants/payload/reapi-recipe.json index 500daa5e5..0240e37f9 100644 --- a/cases/stackstorm-pants/payload/reapi-recipe.json +++ b/cases/stackstorm-pants/payload/reapi-recipe.json @@ -4,6 +4,9 @@ "pants", "--no-pantsd", "--stats-log", + "--pytest-install-from-resolve=pants-plugins", + "--pytest-requirements=['pytest==7.0.1']", + "--no-python-infer-conftests", "test", "--report", "pants-plugins/pack_metadata/target_types_test.py" diff --git a/cases/stackstorm-pants/proposal.md b/cases/stackstorm-pants/proposal.md index 0bc7ea98e..d3911d471 100644 --- a/cases/stackstorm-pants/proposal.md +++ b/cases/stackstorm-pants/proposal.md @@ -8,7 +8,11 @@ Source: [StackStorm/st2 at `9824de4dfd0c869869e310dee729308f398ad83a`](https://g This screen selects `pants-plugins/pack_metadata/target_types_test.py`, a plugin test file that does not require MongoDB, RabbitMQ or Redis services. Pants 2.25.0 and Python 3.11.14 are pinned. Test reports must contain tests and no failures or errors. Full StackStorm service tests are outside this screen. -Declared command: `pants --no-pantsd --stats-log test --report pants-plugins/pack_metadata/target_types_test.py`. +The command selects pytest from the existing `pants-plugins` lockfile and disables +root conftest inference for this isolated plugin test. The normal global pytest +resolve selects the service lockfile, whose unpinned Orquesta Git dependency no +longer matches its retained hash. This screen does not regenerate or relax that +service lockfile. The full command is recorded in `payload/reapi-recipe.json`. The case uses the shared `reapi-fresh-benchmark.yml` workflow, registry runner, source verifier and canonical reporter. Tool installation and dependency diff --git a/results/stackstorm-pants/registry-screening-04/series.json b/results/stackstorm-pants/registry-screening-04/series.json new file mode 100644 index 000000000..040775f98 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-04", + "created_at": "2026-10-05T11:25:33Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "e085a1693be2da021182976715305294e9d52fdc467d0e5f4b3b174b1a78d1d7" +} From c848083da1f4febbae09a6ac66257da43810accc Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:26:02 +0100 Subject: [PATCH 09/22] Retain repeat dispatches and declare OpenCut qualification --- .../1-reapi-fresh-benchmark.yml.json | 35 ++++++++++++++ .../2-reapi-fresh-benchmark.yml.json | 35 ++++++++++++++ .../1-reapi-fresh-benchmark.yml.json | 36 ++++++++++++++ .../2-reapi-fresh-benchmark.yml.json | 36 ++++++++++++++ .../registry-qualification-01/series.json | 48 +++++++++++++++++++ .../1-reapi-fresh-benchmark.yml.json | 35 ++++++++++++++ 6 files changed, 225 insertions(+) create mode 100644 results/gogs-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/gogs-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json create mode 100644 results/msgpack-sbt/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/msgpack-sbt/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json create mode 100644 results/opencut-moon/registry-qualification-01/series.json create mode 100644 results/stackstorm-pants/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json diff --git a/results/gogs-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json b/results/gogs-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..fdb5afc61 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:24:18Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302711581, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302711581" +} diff --git a/results/gogs-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json b/results/gogs-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..0e47102cc --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "gogs-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:24:20Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302715178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302715178" +} diff --git a/results/msgpack-sbt/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json b/results/msgpack-sbt/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..7232c84b8 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:24:22Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302718511, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302718511" +} diff --git a/results/msgpack-sbt/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json b/results/msgpack-sbt/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..6b603b030 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:24:24Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302721578, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302721578" +} diff --git a/results/opencut-moon/registry-qualification-01/series.json b/results/opencut-moon/registry-qualification-01/series.json new file mode 100644 index 000000000..f4b2bc3f6 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "created_at": "2026-10-05T11:26:02Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "variant": null, + "workflow_inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "726ca855e2aad904fe6a34e52fb84e252e35aa6504556537fa0f7c979f9c0bc7" +} diff --git a/results/stackstorm-pants/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json b/results/stackstorm-pants/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..6a8bcafde --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:26:00Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302896058, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302896058" +} From c99edae5024cbe6e0f845f4b3e34fc691c671bc9 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:32:57 +0100 Subject: [PATCH 10/22] Enable Buck2 action uploads and report observed cache hits --- cases/executorch-buck2/cache.patch | 10 ++++ cases/executorch-buck2/case.json | 3 +- .../payload/recipe-contract.json | 3 +- cases/executorch-buck2/proposal.md | 4 ++ .../registry-screening-04/series.json | 49 +++++++++++++++++++ .../registry-qualification-01/series.json | 48 ++++++++++++++++++ scripts/reapi-client.rb | 7 ++- scripts/reapi-setup.rb | 3 +- test/reapi_report_test.rb | 47 ++++++++++++++++++ 9 files changed, 169 insertions(+), 5 deletions(-) create mode 100644 cases/executorch-buck2/cache.patch create mode 100644 results/executorch-buck2/registry-screening-04/series.json create mode 100644 results/stackstorm-pants/registry-qualification-01/series.json create mode 100644 test/reapi_report_test.rb diff --git a/cases/executorch-buck2/cache.patch b/cases/executorch-buck2/cache.patch new file mode 100644 index 000000000..c62b0e6ee --- /dev/null +++ b/cases/executorch-buck2/cache.patch @@ -0,0 +1,10 @@ +--- a/shim_et/xplat/executorch/build/runtime_wrapper.bzl ++++ b/shim_et/xplat/executorch/build/runtime_wrapper.bzl +@@ -301,6 +301,7 @@ + return kwargs + + def _patch_kwargs_cxx(kwargs): ++ kwargs["allow_cache_upload"] = True + env.patch_platforms(kwargs) + env.remove_platform_specific_args(kwargs) + return _patch_kwargs_common(kwargs) diff --git a/cases/executorch-buck2/case.json b/cases/executorch-buck2/case.json index 1d5ac91e2..068382899 100644 --- a/cases/executorch-buck2/case.json +++ b/cases/executorch-buck2/case.json @@ -13,7 +13,8 @@ "kind": "snapshot", "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" } - ] + ], + "patch": "cache.patch" }, "origin": { "repository": "pytorch/executorch", diff --git a/cases/executorch-buck2/payload/recipe-contract.json b/cases/executorch-buck2/payload/recipe-contract.json index 7c5ee1585..6e4353d3b 100644 --- a/cases/executorch-buck2/payload/recipe-contract.json +++ b/cases/executorch-buck2/payload/recipe-contract.json @@ -2,7 +2,8 @@ "upstream_files": { ".buckconfig": "047f2c89c6b718041202195a60039d8ea68f2015ecd084607966e65f2d36d003", ".ci/docker/ci_commit_pins/buck2.txt": "399b4fb63b3401056b204940ac21a2c6ccb5f063728984ab543f05214ef007f3", - "examples/portable/executor_runner/targets.bzl": "6a3a884268f037cc4aeb2187e6d99a9aa27ba3d5216e7f31e4e5bb4c543572c1" + "examples/portable/executor_runner/targets.bzl": "6a3a884268f037cc4aeb2187e6d99a9aa27ba3d5216e7f31e4e5bb4c543572c1", + "shim_et/xplat/executorch/build/runtime_wrapper.bzl": "5acae575afbb23741c95c7c953daa180dd938247eb5d43ecc87b0e85255d337d" }, "commands": {} } diff --git a/cases/executorch-buck2/proposal.md b/cases/executorch-buck2/proposal.md index 1d23862b2..629dd5081 100644 --- a/cases/executorch-buck2/proposal.md +++ b/cases/executorch-buck2/proposal.md @@ -20,6 +20,10 @@ source verifier and canonical reporter. Tool installation and dependency preparation run before timing. Required dependency tasks that the native client runs again remain inside the command measurement. +C++ rules explicitly enable cache uploads through the upstream runtime wrapper. +The execution platform allows uploads only during the cold phase; generator +rules retain their upstream cache policy. + ## Cache and verification BoringCache runs through `boringcache ci run` and `cache-registry --reapi-port`. diff --git a/results/executorch-buck2/registry-screening-04/series.json b/results/executorch-buck2/registry-screening-04/series.json new file mode 100644 index 000000000..f39d71df2 --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "created_at": "2026-10-05T11:32:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "5ce3b7d62703030a03f65e84bd843034d8faa022481a212d964c3dfbb85931b7", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "b8d203ceb51e1632633cc43c486fd493c4effe513931a84e984cbebcad7c6257" +} diff --git a/results/stackstorm-pants/registry-qualification-01/series.json b/results/stackstorm-pants/registry-qualification-01/series.json new file mode 100644 index 000000000..fcbe5c5f3 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/series.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "created_at": "2026-10-05T11:32:57Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "variant": null, + "workflow_inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "ae3ee8ca9d1fcd7b05d186e12a52dd03b42eebaf92f9480c27ea9bd632ac8a58" +} diff --git a/scripts/reapi-client.rb b/scripts/reapi-client.rb index bf9f90b96..eef96b1ef 100644 --- a/scripts/reapi-client.rb +++ b/scripts/reapi-client.rb @@ -64,8 +64,7 @@ def self.verify raise Error, "Output check failed: #{command.first}" unless system(*command) end if ENV.fetch("PHASE") == "warm" - log = File.read("reapi-evidence/build.log") - raise Error, "No native remote-cache hit was recorded" unless Regexp.new(recipe.fetch("warm_hit_pattern"), Regexp::IGNORECASE).match?(log) + raise Error, "No native remote-cache hit was recorded" unless remote_cache_hit? end output = files.to_h { |path| [path, Digest::SHA256.file(path).hexdigest] } File.write("reapi-evidence/outputs.json", JSON.pretty_generate(output) + "\n") @@ -74,6 +73,10 @@ def self.verify raise Error, "Outputs differ from the cold build" unless seed == output end end + + def self.remote_cache_hit? + Regexp.new(recipe.fetch("warm_hit_pattern"), Regexp::IGNORECASE).match?(File.read("reapi-evidence/build.log")) + end end if $PROGRAM_NAME == __FILE__ diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 9ae0f8e0a..cf1785e17 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -70,7 +70,8 @@ def self.report "--workspace", "boringcache/benchmarks", "--cache-tag", BenchmarkPhase.scope(ENV), "--source-repository", context.dig("source", "repository"), "--source-sha", context.dig("source", "revision"), "--build-seconds", timings.fetch("build_seconds").to_s, "--restore-or-setup-seconds", timings.fetch("restore_or_setup_seconds").to_s, - "--save-seconds", timings.fetch("save_seconds").to_s, "--cache-hit", (ENV.fetch("PHASE") == "warm").to_s] + "--save-seconds", timings.fetch("save_seconds").to_s] + args += ["--cache-hit", "true"] if ReapiClient.remote_cache_hit? if ENV.fetch("PROVIDER") == "boringcache" version, status = Open3.capture2("boringcache", "--version") raise "Cannot read CLI version" unless status.success? diff --git a/test/reapi_report_test.rb b/test/reapi_report_test.rb new file mode 100644 index 000000000..4294ef96f --- /dev/null +++ b/test/reapi_report_test.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require "tmpdir" +require "fileutils" +require "json" +require "open3" +require "rbconfig" + +class ReapiReportTest < Minitest::Test + def test_phase_name_does_not_establish_a_cache_hit_or_miss + assert_nil reported_cache_hit("cold", "0 remote cache hits") + assert_nil reported_cache_hit("warm", "0 remote cache hits") + assert_equal true, reported_cache_hit("cold", "2 remote cache hits") + assert_equal true, reported_cache_hit("warm", "81 remote cache hits") + end + + private + + def reported_cache_hit(phase, log) + previous = ENV.to_h + ENV.update("PHASE" => phase, "PROVIDER" => "bazel-remote", "BENCHMARK_ID" => "msgpack-sbt", + "CACHE_LANE" => "fresh", "BENCHMARK_SAMPLE" => "1", + "GITHUB_RUN_ID" => "1", "GITHUB_RUN_ATTEMPT" => "1") + ENV.delete("BENCHMARK_SERIES_ID") + Dir.mktmpdir do |directory| + Dir.chdir(directory) do + FileUtils.mkdir_p("reapi-evidence") + File.write("benchmark-context.json", JSON.generate("source" => {"repository" => "msgpack/msgpack-java", "revision" => "a" * 40})) + File.write("reapi-recipe.json", JSON.generate("warm_hit_pattern" => "\\b[1-9][0-9]* remote cache hits\\b")) + File.write("reapi-evidence/build.log", log) + File.write("reapi-evidence/timing.json", JSON.generate("build_seconds" => 2, "restore_or_setup_seconds" => 1, "save_seconds" => 1)) + source = File.expand_path("../scripts", __dir__) + %w[reapi-setup reapi-client reapi-registry benchmark-phase benchmark-plan].each do |name| + FileUtils.cp(File.join(source, "#{name}.rb"), "#{name}.rb") + end + FileUtils.cp(File.join(source, "canonical/benchmark-report.rb"), "benchmark-report.rb") + output, status = Open3.capture2e(RbConfig.ruby, "reapi-setup.rb", "report") + assert status.success?, output + record = JSON.parse(File.read(Dir["benchmark-results/*.json"].fetch(0))) + record.dig("cache", "hit") + end + end + ensure + ENV.replace(previous) + end +end From 53c7e37db7f7655990fc976badaa65c0fa5bced7 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:33:56 +0100 Subject: [PATCH 11/22] Declare repeated Nix checks and retain native dispatches --- .../1-reapi-fresh-benchmark.yml.json | 36 +++++++++++++ .../canary-qualification-01/series.json | 50 +++++++++++++++++++ .../1-reapi-fresh-benchmark.yml.json | 35 +++++++++++++ .../2-reapi-fresh-benchmark.yml.json | 35 +++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 results/executorch-buck2/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/helix-nix/canary-qualification-01/series.json create mode 100644 results/stackstorm-pants/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/stackstorm-pants/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json diff --git a/results/executorch-buck2/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json b/results/executorch-buck2/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..1485c3b2a --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:33:09Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37303662752, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752" +} diff --git a/results/helix-nix/canary-qualification-01/series.json b/results/helix-nix/canary-qualification-01/series.json new file mode 100644 index 000000000..94aa9db27 --- /dev/null +++ b/results/helix-nix/canary-qualification-01/series.json @@ -0,0 +1,50 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "canary-qualification-01", + "created_at": "2026-10-05T11:33:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "c3a3dcec6a8465f0634261b7d62ffc766c1d7f5a5ddcebbdc1dd618b405dba51", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "3bddda0d7d318cd12969b28360ee49e7582621978f904b5041b480c2ff4d0a25" +} diff --git a/results/stackstorm-pants/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json b/results/stackstorm-pants/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..1f2fb1c0d --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:33:11Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37303666267, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303666267" +} diff --git a/results/stackstorm-pants/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json b/results/stackstorm-pants/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..6cc504869 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:33:12Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37303670119, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303670119" +} From 285eb0d0ad866c46e67e616f6210a7906dcdab9f Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:39:27 +0100 Subject: [PATCH 12/22] Pin Buck2 with the cache permission digest fix --- cases/executorch-buck2/proposal.md | 9 +++- .../registry-screening-05/series.json | 49 +++++++++++++++++++ scripts/reapi-setup.rb | 4 +- 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 results/executorch-buck2/registry-screening-05/series.json diff --git a/cases/executorch-buck2/proposal.md b/cases/executorch-buck2/proposal.md index 629dd5081..eb67595ba 100644 --- a/cases/executorch-buck2/proposal.md +++ b/cases/executorch-buck2/proposal.md @@ -10,8 +10,13 @@ Preparation installs the pinned PyTorch 2.14.0 CPU wheel, verified by its publis SHA-256, for the upstream `torchgen_files` generator. It does not install CUDA dependencies. - -The upstream portable executor runner target and Buck2 2025-05-06 pin are retained. The payload adds a local execution platform with remote caching enabled and remote execution disabled. Source submodules retain upstream pins. Verification checks executable ELF output and cold/warm hashes; it does not run an exported model. +The upstream portable executor runner target is retained. Both providers use +Buck2 2025-06-01 instead of the upstream 2025-05-06 pin. The newer client includes +the [permission-check digest fix](https://github.com/facebook/buck2/pull/946); +the old client sends SHA-1 blobs to the SHA-256 cache during its upload check. +The payload adds a local execution platform with remote caching enabled and +remote execution disabled. Source submodules retain upstream pins. Verification +checks executable ELF output and cold/warm hashes; it does not run an exported model. Declared command: `buck2 build //examples/portable/executor_runner:executor_runner --show-output`. diff --git a/results/executorch-buck2/registry-screening-05/series.json b/results/executorch-buck2/registry-screening-05/series.json new file mode 100644 index 000000000..dc2b9c6d1 --- /dev/null +++ b/results/executorch-buck2/registry-screening-05/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "created_at": "2026-10-05T11:39:27Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "a5fcff27c94f6fb115ea5aff07d0cbb1b571d30f7966c7ada60700a2e22b66f2", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "10fca1a272ec3569d72de5bc0ed5a687263e999a856cbd9181cd3dddca073f25" +} diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index cf1785e17..27393d5b1 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -47,8 +47,8 @@ def self.install end case ReapiClient.recipe.fetch("tool") when "buck2" - installed << download("facebook/buck2", "2025-05-06", "buck2-x86_64-unknown-linux-musl.zst", "buck2", - sha256: "affe6dc6c63bfe2d1f41a42c1f8d187408520ff6d7b6be5a1a502ec5a3acf238") + installed << download("facebook/buck2", "2025-06-01", "buck2-x86_64-unknown-linux-musl.zst", "buck2", + sha256: "7127a64ce1341b2df4d85fd656dd5200c06777512edc605005e362d5f485de76") when "pants" installed << download("pantsbuild/scie-pants", "v0.13.2", "scie-pants-linux-x86_64", "pants", sha256: "74a1e53bc50d6ef6ce1bc67bd9f7b48e549505e0a2453ad4d5ccbc72b0bea874") From 7697aae8e628cb24f758ed99b7be4eb05564fa36 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:40:26 +0100 Subject: [PATCH 13/22] Retain native cache repeats and original failed screens --- data/latest/series.json | 8361 +++++++++++------ migration/evidence.json | 252 + .../completions/37302007157.json | 17 + .../registry-screening-02/report.json | 78 + .../registry-screening-02/report.md | 38 + .../completions/37302678892.json | 16 + .../registry-screening-03/report.json | 222 + .../registry-screening-03/report.md | 38 + .../runs/1-boringcache-cold.json | 141 + .../1-reapi-fresh-benchmark.yml.json | 36 + .../completions/37302711581.json | 13 + .../completions/37302715178.json | 13 + .../registry-qualification-01/report.json | 1202 +++ .../registry-qualification-01/report.md | 37 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + .../runs/2-bazel-remote-cold.json | 139 + .../runs/2-bazel-remote-warm.json | 139 + .../runs/2-boringcache-cold.json | 141 + .../runs/2-boringcache-warm.json | 141 + .../completions/37301997879.json | 13 + .../registry-screening-02/report.json | 629 ++ .../gogs-moon/registry-screening-02/report.md | 33 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + .../completions/37296157157.json | 13 + ...1-grpc-grpc-bazel-fresh-benchmark.yml.json | 33 + results/grpc/output-download-01/report.json | 1087 +++ results/grpc/output-download-01/report.md | 33 + .../runs/1-actions-cache-cold.json | 157 + .../runs/1-actions-cache-warm.json | 157 + .../runs/1-boringcache-cold.json | 184 + .../runs/1-boringcache-warm.json | 184 + .../runs/1-buildbuddy-cold.json | 141 + .../runs/1-buildbuddy-warm.json | 141 + results/grpc/output-download-01/series.json | 45 + .../dispatches/1-nix-fresh-benchmark.yml.json | 36 + .../dispatches/2-nix-fresh-benchmark.yml.json | 36 + .../completions/37302718511.json | 13 + .../completions/37302721578.json | 13 + .../registry-qualification-01/report.json | 1210 +++ .../registry-qualification-01/report.md | 37 + .../runs/1-bazel-remote-cold.json | 140 + .../runs/1-bazel-remote-warm.json | 140 + .../runs/1-boringcache-cold.json | 142 + .../runs/1-boringcache-warm.json | 142 + .../runs/2-bazel-remote-cold.json | 140 + .../runs/2-bazel-remote-warm.json | 140 + .../runs/2-boringcache-cold.json | 142 + .../runs/2-boringcache-warm.json | 142 + .../completions/37302010045.json | 13 + .../registry-screening-03/report.json | 633 ++ .../registry-screening-03/report.md | 33 + .../runs/1-bazel-remote-cold.json | 140 + .../runs/1-bazel-remote-warm.json | 140 + .../runs/1-boringcache-cold.json | 142 + .../runs/1-boringcache-warm.json | 142 + .../completions/37302929873.json | 13 + .../completions/37302932865.json | 13 + .../1-reapi-fresh-benchmark.yml.json | 35 + .../2-reapi-fresh-benchmark.yml.json | 35 + .../registry-qualification-01/report.json | 1202 +++ .../registry-qualification-01/report.md | 37 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + .../runs/2-bazel-remote-cold.json | 139 + .../runs/2-bazel-remote-warm.json | 139 + .../runs/2-boringcache-cold.json | 141 + .../runs/2-boringcache-warm.json | 141 + .../completions/37302001164.json | 16 + .../registry-screening-02/report.json | 506 + .../registry-screening-02/report.md | 40 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../completions/37302476504.json | 13 + .../registry-screening-04/report.json | 629 ++ .../registry-screening-04/report.md | 33 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + .../completions/37303666267.json | 13 + .../completions/37303670119.json | 13 + .../registry-qualification-01/report.json | 1202 +++ .../registry-qualification-01/report.md | 37 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + .../runs/2-bazel-remote-cold.json | 139 + .../runs/2-bazel-remote-warm.json | 139 + .../runs/2-boringcache-cold.json | 141 + .../runs/2-boringcache-warm.json | 141 + .../completions/37302004103.json | 17 + .../registry-screening-02/report.json | 78 + .../registry-screening-02/report.md | 38 + .../completions/37302345686.json | 17 + .../registry-screening-03/report.json | 78 + .../registry-screening-03/report.md | 38 + .../completions/37302896058.json | 13 + .../registry-screening-04/report.json | 629 ++ .../registry-screening-04/report.md | 33 + .../runs/1-bazel-remote-cold.json | 139 + .../runs/1-bazel-remote-warm.json | 139 + .../runs/1-boringcache-cold.json | 141 + .../runs/1-boringcache-warm.json | 141 + 113 files changed, 24511 insertions(+), 2756 deletions(-) create mode 100644 results/executorch-buck2/registry-screening-02/completions/37302007157.json create mode 100644 results/executorch-buck2/registry-screening-02/report.json create mode 100644 results/executorch-buck2/registry-screening-02/report.md create mode 100644 results/executorch-buck2/registry-screening-03/completions/37302678892.json create mode 100644 results/executorch-buck2/registry-screening-03/report.json create mode 100644 results/executorch-buck2/registry-screening-03/report.md create mode 100644 results/executorch-buck2/registry-screening-03/runs/1-boringcache-cold.json create mode 100644 results/executorch-buck2/registry-screening-05/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/gogs-moon/registry-qualification-01/completions/37302711581.json create mode 100644 results/gogs-moon/registry-qualification-01/completions/37302715178.json create mode 100644 results/gogs-moon/registry-qualification-01/report.json create mode 100644 results/gogs-moon/registry-qualification-01/report.md create mode 100644 results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-cold.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-warm.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-cold.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-warm.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/2-boringcache-cold.json create mode 100644 results/gogs-moon/registry-qualification-01/runs/2-boringcache-warm.json create mode 100644 results/gogs-moon/registry-screening-02/completions/37301997879.json create mode 100644 results/gogs-moon/registry-screening-02/report.json create mode 100644 results/gogs-moon/registry-screening-02/report.md create mode 100644 results/gogs-moon/registry-screening-02/runs/1-bazel-remote-cold.json create mode 100644 results/gogs-moon/registry-screening-02/runs/1-bazel-remote-warm.json create mode 100644 results/gogs-moon/registry-screening-02/runs/1-boringcache-cold.json create mode 100644 results/gogs-moon/registry-screening-02/runs/1-boringcache-warm.json create mode 100644 results/grpc/output-download-01/completions/37296157157.json create mode 100644 results/grpc/output-download-01/dispatches/1-grpc-grpc-bazel-fresh-benchmark.yml.json create mode 100644 results/grpc/output-download-01/report.json create mode 100644 results/grpc/output-download-01/report.md create mode 100644 results/grpc/output-download-01/runs/1-actions-cache-cold.json create mode 100644 results/grpc/output-download-01/runs/1-actions-cache-warm.json create mode 100644 results/grpc/output-download-01/runs/1-boringcache-cold.json create mode 100644 results/grpc/output-download-01/runs/1-boringcache-warm.json create mode 100644 results/grpc/output-download-01/runs/1-buildbuddy-cold.json create mode 100644 results/grpc/output-download-01/runs/1-buildbuddy-warm.json create mode 100644 results/grpc/output-download-01/series.json create mode 100644 results/helix-nix/canary-qualification-01/dispatches/1-nix-fresh-benchmark.yml.json create mode 100644 results/helix-nix/canary-qualification-01/dispatches/2-nix-fresh-benchmark.yml.json create mode 100644 results/msgpack-sbt/registry-qualification-01/completions/37302718511.json create mode 100644 results/msgpack-sbt/registry-qualification-01/completions/37302721578.json create mode 100644 results/msgpack-sbt/registry-qualification-01/report.json create mode 100644 results/msgpack-sbt/registry-qualification-01/report.md create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-cold.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-warm.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-cold.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-warm.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-cold.json create mode 100644 results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-warm.json create mode 100644 results/msgpack-sbt/registry-screening-03/completions/37302010045.json create mode 100644 results/msgpack-sbt/registry-screening-03/report.json create mode 100644 results/msgpack-sbt/registry-screening-03/report.md create mode 100644 results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-cold.json create mode 100644 results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-warm.json create mode 100644 results/msgpack-sbt/registry-screening-03/runs/1-boringcache-cold.json create mode 100644 results/msgpack-sbt/registry-screening-03/runs/1-boringcache-warm.json create mode 100644 results/opencut-moon/registry-qualification-01/completions/37302929873.json create mode 100644 results/opencut-moon/registry-qualification-01/completions/37302932865.json create mode 100644 results/opencut-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json create mode 100644 results/opencut-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json create mode 100644 results/opencut-moon/registry-qualification-01/report.json create mode 100644 results/opencut-moon/registry-qualification-01/report.md create mode 100644 results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-cold.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-warm.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-cold.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-warm.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/2-boringcache-cold.json create mode 100644 results/opencut-moon/registry-qualification-01/runs/2-boringcache-warm.json create mode 100644 results/opencut-moon/registry-screening-02/completions/37302001164.json create mode 100644 results/opencut-moon/registry-screening-02/report.json create mode 100644 results/opencut-moon/registry-screening-02/report.md create mode 100644 results/opencut-moon/registry-screening-02/runs/1-bazel-remote-cold.json create mode 100644 results/opencut-moon/registry-screening-02/runs/1-bazel-remote-warm.json create mode 100644 results/opencut-moon/registry-screening-02/runs/1-boringcache-cold.json create mode 100644 results/opencut-moon/registry-screening-04/completions/37302476504.json create mode 100644 results/opencut-moon/registry-screening-04/report.json create mode 100644 results/opencut-moon/registry-screening-04/report.md create mode 100644 results/opencut-moon/registry-screening-04/runs/1-bazel-remote-cold.json create mode 100644 results/opencut-moon/registry-screening-04/runs/1-bazel-remote-warm.json create mode 100644 results/opencut-moon/registry-screening-04/runs/1-boringcache-cold.json create mode 100644 results/opencut-moon/registry-screening-04/runs/1-boringcache-warm.json create mode 100644 results/stackstorm-pants/registry-qualification-01/completions/37303666267.json create mode 100644 results/stackstorm-pants/registry-qualification-01/completions/37303670119.json create mode 100644 results/stackstorm-pants/registry-qualification-01/report.json create mode 100644 results/stackstorm-pants/registry-qualification-01/report.md create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-cold.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-warm.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-cold.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-warm.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-cold.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-warm.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-cold.json create mode 100644 results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-warm.json create mode 100644 results/stackstorm-pants/registry-screening-02/completions/37302004103.json create mode 100644 results/stackstorm-pants/registry-screening-02/report.json create mode 100644 results/stackstorm-pants/registry-screening-02/report.md create mode 100644 results/stackstorm-pants/registry-screening-03/completions/37302345686.json create mode 100644 results/stackstorm-pants/registry-screening-03/report.json create mode 100644 results/stackstorm-pants/registry-screening-03/report.md create mode 100644 results/stackstorm-pants/registry-screening-04/completions/37302896058.json create mode 100644 results/stackstorm-pants/registry-screening-04/report.json create mode 100644 results/stackstorm-pants/registry-screening-04/report.md create mode 100644 results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-cold.json create mode 100644 results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-warm.json create mode 100644 results/stackstorm-pants/registry-screening-04/runs/1-boringcache-cold.json create mode 100644 results/stackstorm-pants/registry-screening-04/runs/1-boringcache-warm.json diff --git a/data/latest/series.json b/data/latest/series.json index b00899315..fa14af620 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -2413,230 +2413,237 @@ "runs_without_archives": [] }, { - "case_id": "grpc", - "series_id": "fresh-canary-repair-01", - "plan_path": "results/grpc/fresh-canary-repair-01/series.json", + "case_id": "executorch-buck2", + "series_id": "registry-screening-01", + "plan_path": "results/executorch-buck2/registry-screening-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", "providers": [ "boringcache", - "actions-cache", - "buildbuddy" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "grpc/grpc", + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", "pins": [ { - "path": "upstream", - "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" } ] }, - "plan_sha256": "3bad3fe86f24dd2597d4fe3052b82d5b5144b705d6265f219b67a9f2820f238e", - "definition_sha256": "c427f90c3baea4391aef672f3a11f126a4fa812a7a76f1a9928e74b5a375a39e", - "state": "failed", + "plan_sha256": "42c1b64a7a5085d9eff94d9dc59d3478767045b9375686358e6c88ae98364de8", + "definition_sha256": "41f28f4fc5f029e5ae30496d77ed4c4410eae41389bc54550caf802bf3cae1ce", + "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], [ 1, "warm", "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" ] ], - "summaries": [ - { - "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 3281, - "min": 3281, - "max": 3281 - }, - "storage": { - "median": 163728649, - "min": 163728649, - "max": 163728649 - }, - "storage_measured_count": 1 - }, - { - "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 72, - "min": 72, - "max": 72 - }, - "storage": { - "median": 163728649, - "min": 163728649, - "max": 163728649 - }, - "storage_measured_count": 1 - }, - { - "phase": "cold", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 2144, - "min": 2144, - "max": 2144 - }, - "storage": { - "median": 632637546, - "min": 632637546, - "max": 632637546 - }, - "storage_measured_count": 1 - }, - { - "phase": "cold", - "provider": "buildbuddy", - "count": 1, - "measurement": { - "median": 2152, - "min": 2152, - "max": 2152 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "warm", - "provider": "buildbuddy", - "count": 1, - "measurement": { - "median": 135, - "min": 135, - "max": 135 - }, - "storage": null, - "storage_measured_count": 0 - } + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "executorch-buck2", + "series_id": "registry-screening-02", + "plan_path": "results/executorch-buck2/registry-screening-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "plan_sha256": "a5eda8a465657d35135c8cfb6188a55cffb58a62c7b207ce9e0ae265529dd667", + "definition_sha256": "e48b555d2371f86823c65e153c076ce13cf367ad1f37f51d67e5a205d43a2b74", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] ], + "summaries": [], "dispatches": [ { - "case_id": "grpc", + "case_id": "executorch-buck2", "repository": "boringcache/benchmarks", - "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "cli_version": "vcli-canary-4c236cd7ee00", - "series_id": "fresh-canary-repair-01", + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", "sample": "1" }, - "ref": "independent-benchmark-samples", + "ref": "main", "state": "requested", - "requested_at": "2026-10-05T09:10:05Z", - "run_id": 37288249894, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288249894" + "requested_at": "2026-10-05T11:17:38Z", + "run_id": 37302007157, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157" } ], "completions": [ { "schema_version": 1, - "case_id": "grpc", - "series_id": "fresh-canary-repair-01", + "case_id": "executorch-buck2", + "series_id": "registry-screening-02", "sample": 1, - "run_id": "37288249894", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288249894", - "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", + "run_id": "37302007157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", "github_conclusion": "failure", "status": "failed", "errors": [ "Workflow concluded failure", - "BoringCache gRPC Bazel warm: failure" + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" ], - "evidence_manifest_sha256": "2b0066c0d1a1fb732ab879c3b888b0d12b311252a6c17fdd222280e5b51e9cc5" + "evidence_manifest_sha256": "475ddd9b7f53e3a6b7ced765ccacb53fe1bdc2d46d2001bd7b28c4c4c94f8cc0" } ], "report_state": "current", - "report_path": "results/grpc/fresh-canary-repair-01/report.json", + "report_path": "results/executorch-buck2/registry-screening-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37288249894, + "run_id": 37302007157, "inventory_state": "complete", - "verified_files": 14, + "verified_files": 9, "gaps": [], "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288249894.tar.gz", - "bundle_bytes": 2441898, - "sha256": "fc2efcfa0277f1c4bf88ea4d10d9a01e23141e714b77c54e7d2553cfb37f0365", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302007157.tar.gz", + "bundle_bytes": 122488, + "sha256": "08b74d110cb201e2f8cefd83be941d7e3b2e63ea1cce174edacebff156e2ffcd", "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "grpc", - "series_id": "local-outputs-01", - "plan_path": "results/grpc/local-outputs-01/series.json", + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "plan_path": "results/executorch-buck2/registry-screening-03/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", "providers": [ "boringcache", - "actions-cache", - "buildbuddy" + "bazel-remote" ], - "timed_scope": "Bazel build with local top-level outputs plus measured cache restore/setup; output checks and post-job cache save are outside build timing", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "grpc/grpc", + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", "pins": [ { - "path": "upstream", - "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" } ] }, - "plan_sha256": "eec4dd52b91d8f3cd6698c0c4be58b3c8b1643777cfd72ddebebb1f11d054b25", - "definition_sha256": "3ba398321d9ad7a08f2517505bc3762cb29463f172c824b254de38e59e02cc6d", - "state": "requested", + "plan_sha256": "2ded7301f2b56ac93494c09c0bb722d6c01766e1528c760f6af15cbce49b25bf", + "definition_sha256": "29364c9b6c1bdf023476a589a9b69a641cee3d1264058927f4e8d8ddd7592ed9", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ [ 1, "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "cold", - "buildbuddy" + "bazel-remote" ], [ 1, @@ -2646,78 +2653,115 @@ [ 1, "warm", - "actions-cache" - ], - [ - 1, - "warm", - "buildbuddy" + "bazel-remote" ] ], - "summaries": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], "dispatches": [ { - "case_id": "grpc", + "case_id": "executorch-buck2", "repository": "boringcache/benchmarks", - "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { + "case_id": "executorch-buck2", "cli_version": "vcli-canary-7a5b27146ebe", - "series_id": "local-outputs-01", + "series_id": "registry-screening-03", "sample": "1" }, - "ref": "grpc-local-outputs", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-05T10:19:05Z", - "run_id": 37295807809, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37295807809" + "requested_at": "2026-10-05T11:24:00Z", + "run_id": 37302678892, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "evidence": [], - "runs_without_archives": [ - "37295807809" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302678892", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892", + "source_sha": "829fa130f334a8af9df0c3312027921e2f2caaec", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "fda433158d3ad68c400a99c3b8a7e3f6defa9bcca370e4625e907d34853ecc9b" + } + ], + "report_state": "current", + "report_path": "results/executorch-buck2/registry-screening-03/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37302678892, + "inventory_state": "complete", + "verified_files": 10, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302678892.tar.gz", + "bundle_bytes": 106242, + "sha256": "478596b3419d210ba53738011b8decda9402aff7ab897415be11aa7e11b65629", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] }, { - "case_id": "helix-nix", - "series_id": "canary-screening-01", - "plan_path": "results/helix-nix/canary-screening-01/series.json", + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "plan_path": "results/executorch-buck2/registry-screening-04/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "Does Nix restore the Helix package closure on a fresh store?", + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", "providers": [ "boringcache", - "cachix" + "bazel-remote" ], - "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", - "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", "sample_count": 2 }, "source": { - "repository": "helix-editor/helix", - "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", "pins": [ { "path": "proposal.md", "kind": "snapshot", - "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" } - ] + ], + "patch": "cache.patch" }, - "plan_sha256": "13b528ec33fad1168b25a3aba84ad179e64e45d9e34c23cde1e56cf505c35829", - "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", - "state": "planned", + "plan_sha256": "b8d203ceb51e1632633cc43c486fd493c4effe513931a84e984cbebcad7c6257", + "definition_sha256": "5ce3b7d62703030a03f65e84bd843034d8faa022481a212d964c3dfbb85931b7", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ [ 1, @@ -2727,7 +2771,7 @@ [ 1, "cold", - "cachix" + "bazel-remote" ], [ 1, @@ -2737,54 +2781,77 @@ [ 1, "warm", - "cachix" + "bazel-remote" ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", + "sample": "1" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:33:09Z", + "run_id": 37303662752, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752" + } + ], "completions": [], "report_state": "missing", "report_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37303662752" + ] }, { - "case_id": "helix-nix", - "series_id": "canary-screening-02", - "plan_path": "results/helix-nix/canary-screening-02/series.json", + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "plan_path": "results/executorch-buck2/registry-screening-05/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "Does Nix restore the Helix package closure on a fresh store?", + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", "providers": [ "boringcache", - "cachix" + "bazel-remote" ], - "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", - "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", "sample_count": 2 }, "source": { - "repository": "helix-editor/helix", - "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", "pins": [ { "path": "proposal.md", "kind": "snapshot", - "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" } - ] + ], + "patch": "cache.patch" }, - "plan_sha256": "c091853f2c86baae18c7f93abe5e4dc93476624cd5833cab107488beaaacbcb0", - "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", - "state": "planned", + "plan_sha256": "10fca1a272ec3569d72de5bc0ed5a687263e999a856cbd9181cd3dddca073f25", + "definition_sha256": "a5fcff27c94f6fb115ea5aff07d0cbb1b571d30f7966c7ada60700a2e22b66f2", + "state": "requested", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ [ 1, @@ -2794,7 +2861,7 @@ [ 1, "cold", - "cachix" + "bazel-remote" ], [ 1, @@ -2804,409 +2871,407 @@ [ 1, "warm", - "cachix" + "bazel-remote" ] ], "summaries": [], - "dispatches": [], + "dispatches": [ + { + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-05", + "sample": "1" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:39:39Z", + "run_id": 37304358750, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750" + } + ], "completions": [], "report_state": "missing", "report_path": null, "evidence": [], - "runs_without_archives": [] + "runs_without_archives": [ + "37304358750" + ] }, { - "case_id": "hugo-go", - "series_id": "direct-dispatch-qualification", - "plan_path": "results/hugo-go/direct-dispatch-qualification/series.json", + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "plan_path": "results/gogs-moon/registry-qualification-01/series.json", "lane": "fresh", "variant": null, "sample_count": 2, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Moon restore the Gogs web build on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "gohugoio/hugo", + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", "pins": [ { - "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" } ] }, - "plan_sha256": "bfad4beee177fef96c8b7c6eb3aee4df15d33fa69d5b44ba39b5fee5651f0d5b", - "definition_sha256": "c7923cefca6e69c5ade419e595ff7093be51a2df0cda6e14c6c79b7ff6d6b696", + "plan_sha256": "6f0af9bb2ddd8c15c7be1a0810617d8973841bcd8f8ff77dc61ef299041cd5c5", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 2, - "measurement": { - "median": 79.5, - "min": 78, - "max": 81 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 2, - "measurement": { - "median": 17.0, - "min": 11, - "max": 23 - }, - "storage": { - "median": 161851116.5, - "min": 161849774, - "max": 161852459 - }, - "storage_measured_count": 2 + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", "count": 2, - "measurement": { - "median": 76.0, - "min": 72, - "max": 80 - }, - "storage": { - "median": 646497724.5, - "min": 604312219, - "max": 688683230 - }, - "storage_measured_count": 2 + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", "provider": "boringcache", "count": 2, - "measurement": { - "median": 18.0, - "min": 16, - "max": 20 - }, - "storage": { - "median": 811503108.0, - "min": 811503108, - "max": 811503108 - }, - "storage_measured_count": 2 + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "hugo-go", + "case_id": "gogs-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", - "cli_version": "v1.33.0", - "series_id": "direct-dispatch-qualification", + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", "sample": "1" }, - "ref": "benchmark-consolidation-2026-10-02", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-02T17:03:43Z", - "run_id": 37038153301, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038153301" + "requested_at": "2026-10-05T11:24:18Z", + "run_id": 37302711581, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302711581" }, { - "case_id": "hugo-go", + "case_id": "gogs-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", - "cli_version": "v1.33.0", - "series_id": "direct-dispatch-qualification", + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", "sample": "2" }, - "ref": "benchmark-consolidation-2026-10-02", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-02T17:03:46Z", - "run_id": 37038157657, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038157657" + "requested_at": "2026-10-05T11:24:20Z", + "run_id": 37302715178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302715178" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "direct-dispatch-qualification", + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", "sample": 1, - "run_id": "37038153301", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038153301", - "source_sha": "aef10ff570e92395652bc8c5d7a4fe7cfd0bd114", + "run_id": "37302711581", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302711581", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "9fff7441d035ae732e83c862aaa47ac64bbc7b429e840a56a864b21140239510" + "evidence_manifest_sha256": "2481184fb7a464c4746b379d0038961240a6463104da7cf3b73c107adc0e27a5" }, { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "direct-dispatch-qualification", + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", "sample": 2, - "run_id": "37038157657", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038157657", - "source_sha": "aef10ff570e92395652bc8c5d7a4fe7cfd0bd114", + "run_id": "37302715178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302715178", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "65d03d9f9b79f124339cf07c94d7a6cb63a8e5e770e8d84cc9f4d0a5f1fc7cc8" + "evidence_manifest_sha256": "dfe4c6cef48e640a30e0e08613297ccb7201968816fbccad860d0c9f22e117c7" } ], "report_state": "current", - "report_path": "results/hugo-go/direct-dispatch-qualification/report.json", + "report_path": "results/gogs-moon/registry-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37038153301, + "run_id": 37302711581, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 16, "gaps": [], - "scope": "This run's attempts, jobs, logs, artifacts, commit metadata, and workflow; not the original repository history", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-direct-dispatch-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-direct-dispatch-2026-10-02/benchmarks-37038153301.tar", - "bundle_bytes": 303616, - "sha256": "0780c3e940a26b78f835bf094a01f9d65c2490e6d52c7c08dbb6c10ce686e601", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302711581.tar.gz", + "bundle_bytes": 3605287, + "sha256": "a1fff6e0704dbc719fc6f8e54d406828015b7226f51348fc6d8fb5e587be734a", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" }, { "repository": "boringcache/benchmarks", - "run_id": 37038157657, + "run_id": 37302715178, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 16, "gaps": [], - "scope": "This run's attempts, jobs, logs, artifacts, commit metadata, and workflow; not the original repository history", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-direct-dispatch-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-direct-dispatch-2026-10-02/benchmarks-37038157657.tar", - "bundle_bytes": 303616, - "sha256": "66debe7af8c10f0f9ec657aa58d9048029180143dc02eaa0d0d279138bc6b674", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302715178.tar.gz", + "bundle_bytes": 3587778, + "sha256": "361ffdc1d10179e1747213f0b8f17ae4a6d969b86aa1131357131c02f6fde4ae", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "independent-samples-01", - "plan_path": "results/hugo-go/independent-samples-01/series.json", + "case_id": "gogs-moon", + "series_id": "registry-screening-01", + "plan_path": "results/gogs-moon/registry-screening-01/series.json", "lane": "fresh", "variant": null, - "sample_count": 2, + "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Moon restore the Gogs web build on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "gohugoio/hugo", + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", "pins": [ { - "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", - "kind": "gitlink" - } + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } ] }, - "plan_sha256": "fc20058701dd1bf0b6a94acf40bf1d76adda8e37695472b295ec9bcc4b4da562", - "definition_sha256": "21702e83473c3a7a99cae19b7b4aa108243f49e8064878e4632804d315988a52", + "plan_sha256": "693789eeccd6bf2699d1b61cbf4c53d245f53a03314c7814c37fde270d7e9a3e", + "definition_sha256": "d48ebe1844ab9ee2551e9fb207de88cc9335a403841e4ad7c25579faeb481734", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "gogs-moon", + "series_id": "registry-screening-02", + "plan_path": "results/gogs-moon/registry-screening-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "plan_sha256": "470d098106a74bafaa7ac215832052f8406a2d8c86b298be4fbda5b9bc416a9c", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", - "count": 2, - "measurement": { - "median": 70.0, - "min": 61, - "max": 79 - }, + "provider": "bazel-remote", + "count": 1, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", - "count": 2, - "measurement": { - "median": 9.0, - "min": 9, - "max": 9 - }, - "storage": { - "median": 161808547.0, - "min": 161804322, - "max": 161812772 - }, - "storage_measured_count": 2 + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", - "count": 2, - "measurement": { - "median": 87.5, - "min": 85, - "max": 90 - }, - "storage": { - "median": 612837451.5, - "min": 610123303, - "max": 615551600 - }, - "storage_measured_count": 2 + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", "provider": "boringcache", - "count": 2, - "measurement": { - "median": 21.0, - "min": 18, - "max": 24 - }, - "storage": { - "median": 811503108.0, - "min": 811503108, - "max": 811503108 - }, - "storage_measured_count": 2 + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "hugo-go", + "case_id": "gogs-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", - "series_id": "independent-samples-01", + "case_id": "gogs-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", "sample": "1" }, - "ref": "independent-benchmark-samples", - "state": "requested", - "requested_at": "2026-10-05T09:10:00Z", - "run_id": 37288240708, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288240708" - }, - { - "case_id": "hugo-go", - "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", - "lane": "fresh", - "inputs": { - "case_id": "hugo-go", - "series_id": "independent-samples-01", - "sample": "2" - }, - "ref": "independent-benchmark-samples", + "ref": "main", "state": "requested", - "requested_at": "2026-10-05T09:10:02Z", - "run_id": 37288246103, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288246103" + "requested_at": "2026-10-05T11:17:32Z", + "run_id": 37301997879, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37301997879" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "independent-samples-01", + "case_id": "gogs-moon", + "series_id": "registry-screening-02", "sample": 1, - "run_id": "37288240708", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288240708", - "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "4fd9ad410330fc0e34949e2b298d079f188b543286ed7c662e52bcb969d5c925" - }, - { - "schema_version": 1, - "case_id": "hugo-go", - "series_id": "independent-samples-01", - "sample": 2, - "run_id": "37288246103", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288246103", - "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", + "run_id": "37301997879", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37301997879", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "67db601ec3dfd03345ef6540efe4716d76d8398fb6b1d4c1f6dfbfa8737c4909" + "evidence_manifest_sha256": "6590381ed0e9bde4d8e3a5cbf8d4a1efd80ec5dba9fa4689fe64a0b3f886bbe6" } ], "report_state": "current", - "report_path": "results/hugo-go/independent-samples-01/report.json", + "report_path": "results/gogs-moon/registry-screening-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37288240708, - "inventory_state": "complete", - "verified_files": 17, - "gaps": [], - "scope": "One declared Hugo Go sample, including its attempts, jobs, logs, artifacts, commit and workflow", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288240708.tar.gz", - "bundle_bytes": 192719, - "sha256": "5d1eba3f14c045aebe3dd0a8926ca435d8ed5a994794f8050e121a2917621ef3", - "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37288246103, + "run_id": 37301997879, "inventory_state": "complete", - "verified_files": 17, + "verified_files": 16, "gaps": [], - "scope": "One declared Hugo Go sample, including its attempts, jobs, logs, artifacts, commit and workflow", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288246103.tar.gz", - "bundle_bytes": 192849, - "sha256": "9cbbc3f43c428fec36a3d6c642826156335170204c098af683034b5168d7d67a", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37301997879.tar.gz", + "bundle_bytes": 3628592, + "sha256": "56088bc5bf76d8a296beca3cd0ceac2265dc449bd0baf807ddad532a92d21767", "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "merged-main-qualification-03", - "plan_path": "results/hugo-go/merged-main-qualification-03/series.json", + "case_id": "grpc", + "series_id": "fresh-canary-repair-01", + "plan_path": "results/grpc/fresh-canary-repair-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, @@ -3214,7 +3279,8 @@ "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", "providers": [ "boringcache", - "actions-cache" + "actions-cache", + "buildbuddy" ], "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", @@ -3223,49 +3289,59 @@ "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "grpc/grpc", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", "kind": "gitlink" } ] }, - "plan_sha256": "fb020c5ffe80dad97e564738deb8f4192835b3551b3ab025d42c2103ada169c5", - "definition_sha256": "8575b4c7e5c1a048ccb9a3a36f7653474e78b0aea76f6166818bd51560409a39", - "state": "completed", + "plan_sha256": "3bad3fe86f24dd2597d4fe3052b82d5b5144b705d6265f219b67a9f2820f238e", + "definition_sha256": "c427f90c3baea4391aef672f3a11f126a4fa812a7a76f1a9928e74b5a375a39e", + "state": "failed", "publication": "unreviewed", - "valid_for_comparison": true, - "execution_verified": true, + "valid_for_comparison": false, + "execution_verified": false, "methodology_issues": [], - "missing_observations": [], + "missing_observations": [ + [ + 1, + "warm", + "boringcache" + ] + ], "summaries": [ { "phase": "cold", "provider": "actions-cache", "count": 1, "measurement": { - "median": 53, - "min": 53, - "max": 53 + "median": 3281, + "min": 3281, + "max": 3281 }, - "storage": null, - "storage_measured_count": 0 + "storage": { + "median": 163728649, + "min": 163728649, + "max": 163728649 + }, + "storage_measured_count": 1 }, { "phase": "warm", "provider": "actions-cache", "count": 1, "measurement": { - "median": 10, - "min": 10, - "max": 10 + "median": 72, + "min": 72, + "max": 72 }, "storage": { - "median": 161848750, - "min": 161848750, - "max": 161848750 + "median": 163728649, + "min": 163728649, + "max": 163728649 }, "storage_measured_count": 1 }, @@ -3274,91 +3350,101 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 69, - "min": 69, - "max": 69 + "median": 2144, + "min": 2144, + "max": 2144 }, "storage": { - "median": 588257602, - "min": 588257602, - "max": 588257602 + "median": 632637546, + "min": 632637546, + "max": 632637546 }, "storage_measured_count": 1 }, { - "phase": "warm", - "provider": "boringcache", + "phase": "cold", + "provider": "buildbuddy", "count": 1, "measurement": { - "median": 16, - "min": 16, - "max": 16 + "median": 2152, + "min": 2152, + "max": 2152 }, - "storage": { - "median": 811503108, - "min": 811503108, - "max": 811503108 + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "buildbuddy", + "count": 1, + "measurement": { + "median": 135, + "min": 135, + "max": 135 }, - "storage_measured_count": 1 + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "hugo-go", + "case_id": "grpc", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", - "cli_version": "v1.33.0", - "series_id": "merged-main-qualification-03", + "cli_version": "vcli-canary-4c236cd7ee00", + "series_id": "fresh-canary-repair-01", "sample": "1" }, - "ref": "main", + "ref": "independent-benchmark-samples", "state": "requested", - "requested_at": "2026-10-04T17:02:40Z", - "run_id": 37219016891, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219016891" + "requested_at": "2026-10-05T09:10:05Z", + "run_id": 37288249894, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288249894" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "merged-main-qualification-03", + "case_id": "grpc", + "series_id": "fresh-canary-repair-01", "sample": 1, - "run_id": "37219016891", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219016891", - "source_sha": "fac4677ae1b217db78662291b34fa89d6253f92c", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "d7c4ede0806a1b7450b24d025ffa50ff9baec7de1d7e9adf12fd34c1d783a413" + "run_id": "37288249894", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288249894", + "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache gRPC Bazel warm: failure" + ], + "evidence_manifest_sha256": "2b0066c0d1a1fb732ab879c3b888b0d12b311252a6c17fdd222280e5b51e9cc5" } ], "report_state": "current", - "report_path": "results/hugo-go/merged-main-qualification-03/report.json", + "report_path": "results/grpc/fresh-canary-repair-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37219016891, + "run_id": 37288249894, "inventory_state": "complete", - "verified_files": 17, + "verified_files": 14, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-merged-harness-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-merged-harness-2026-10-04/benchmarks-37219016891.tar.gz", - "bundle_bytes": 202782, - "sha256": "33a7ad7853c1ea64a9a923dc74389a5dd850fc6197af5dfa59bc0091891bef06", - "publication_verification": "Downloaded published assets, compared archive digests, extracted each bundle, and verified scoped file inventory and completeness" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288249894.tar.gz", + "bundle_bytes": 2441898, + "sha256": "fc2efcfa0277f1c4bf88ea4d10d9a01e23141e714b77c54e7d2553cfb37f0365", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "shared-evidence-01", - "plan_path": "results/hugo-go/shared-evidence-01/series.json", + "case_id": "grpc", + "series_id": "local-outputs-01", + "plan_path": "results/grpc/local-outputs-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, @@ -3366,8 +3452,105 @@ "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", "providers": [ "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Bazel build with local top-level outputs plus measured cache restore/setup; output checks and post-job cache save are outside build timing", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "eec4dd52b91d8f3cd6698c0c4be58b3c8b1643777cfd72ddebebb1f11d054b25", + "definition_sha256": "3ba398321d9ad7a08f2517505bc3762cb29463f172c824b254de38e59e02cc6d", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", "actions-cache" ], + [ + 1, + "cold", + "buildbuddy" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ], + [ + 1, + "warm", + "buildbuddy" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "grpc", + "repository": "boringcache/benchmarks", + "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "local-outputs-01", + "sample": "1" + }, + "ref": "grpc-local-outputs", + "state": "requested", + "requested_at": "2026-10-05T10:19:05Z", + "run_id": 37295807809, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37295807809" + } + ], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [ + "37295807809" + ] + }, + { + "case_id": "grpc", + "series_id": "output-download-01", + "plan_path": "results/grpc/output-download-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", @@ -3375,17 +3558,17 @@ "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "grpc/grpc", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", "kind": "gitlink" } ] }, - "plan_sha256": "5885c0cb1808992f61acfcdcb136173b0820f554c6d9f0417b10a97f24b1bbb4", - "definition_sha256": "21e92460ce19eeb2e7a57492824a3729ca2e34df345f66e94a0a905df1410e70", + "plan_sha256": "a363a40e474799553b5fbaf158c0a060ab6b2bcee8aa9f9ab6bfc7e216553f50", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", "state": "completed", "publication": "unreviewed", "valid_for_comparison": true, @@ -3398,26 +3581,30 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 52, - "min": 52, - "max": 52 + "median": 3193, + "min": 3193, + "max": 3193 }, - "storage": null, - "storage_measured_count": 0 + "storage": { + "median": 163734007, + "min": 163734007, + "max": 163734007 + }, + "storage_measured_count": 1 }, { "phase": "warm", "provider": "actions-cache", "count": 1, "measurement": { - "median": 8, - "min": 8, - "max": 8 + "median": 93, + "min": 93, + "max": 93 }, "storage": { - "median": 161840391, - "min": 161840391, - "max": 161840391 + "median": 163734007, + "min": 163734007, + "max": 163734007 }, "storage_measured_count": 1 }, @@ -3426,14 +3613,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 98, - "min": 98, - "max": 98 + "median": 2107, + "min": 2107, + "max": 2107 }, "storage": { - "median": 588257602, - "min": 588257602, - "max": 588257602 + "median": 647876056, + "min": 647876056, + "max": 647876056 }, "storage_measured_count": 1 }, @@ -3442,84 +3629,2151 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 17, - "min": 17, - "max": 17 + "median": 84, + "min": 84, + "max": 84 }, "storage": { - "median": 811503108, - "min": 811503108, - "max": 811503108 + "median": 718160974, + "min": 718160974, + "max": 718160974 }, "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "buildbuddy", + "count": 1, + "measurement": { + "median": 2092, + "min": 2092, + "max": 2092 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "buildbuddy", + "count": 1, + "measurement": { + "median": 122, + "min": 122, + "max": 122 + }, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "hugo-go", + "case_id": "grpc", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", - "series_id": "shared-evidence-01", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "output-download-01", "sample": "1" }, - "ref": "simplify-benchmark-cases", + "ref": "run-tool-family-cases", "state": "requested", - "requested_at": "2026-10-05T10:03:05Z", - "run_id": 37294037661, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37294037661" + "requested_at": "2026-10-05T10:22:20Z", + "run_id": 37296157157, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296157157" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "shared-evidence-01", + "case_id": "grpc", + "series_id": "output-download-01", "sample": 1, - "run_id": "37294037661", + "run_id": "37296157157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296157157", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7192d33d863f1e7abb3f4ad9abe8eeb691310cd828a5f6ac2d107c0429bd110e" + } + ], + "report_state": "current", + "report_path": "results/grpc/output-download-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37296157157, + "inventory_state": "complete", + "verified_files": 18, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37296157157.tar.gz", + "bundle_bytes": 2492084, + "sha256": "89cb0aded5b7856075555104bd6e276efdb391534590b13201cf612fd1b8b44b", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "helix-nix", + "series_id": "canary-qualification-01", + "plan_path": "results/helix-nix/canary-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "plan_sha256": "3bddda0d7d318cd12969b28360ee49e7582621978f904b5041b480c2ff4d0a25", + "definition_sha256": "c3a3dcec6a8465f0634261b7d62ffc766c1d7f5a5ddcebbdc1dd618b405dba51", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "cachix" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "cachix" + ], + [ + 2, + "cold", + "boringcache" + ], + [ + 2, + "cold", + "cachix" + ], + [ + 2, + "warm", + "boringcache" + ], + [ + 2, + "warm", + "cachix" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "canary-qualification-01", + "sample": "1" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:34:11Z", + "run_id": 37303778364, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303778364" + }, + { + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "canary-qualification-01", + "sample": "2" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:34:13Z", + "run_id": 37303782303, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303782303" + } + ], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [ + "37303778364", + "37303782303" + ] + }, + { + "case_id": "helix-nix", + "series_id": "canary-screening-01", + "plan_path": "results/helix-nix/canary-screening-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "plan_sha256": "13b528ec33fad1168b25a3aba84ad179e64e45d9e34c23cde1e56cf505c35829", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "cachix" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "cachix" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "helix-nix", + "series_id": "canary-screening-02", + "plan_path": "results/helix-nix/canary-screening-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "plan_sha256": "c091853f2c86baae18c7f93abe5e4dc93476624cd5833cab107488beaaacbcb0", + "definition_sha256": "f8f429de5d5cf14ae336f5691084bebbf5a7c68d93e76fcde5c3bcd702f370ef", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 296.34937598663436, + "min": 296.34937598663436, + "max": 296.34937598663436 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 5.810530557688935, + "min": 5.810530557688935, + "max": 5.810530557688935 + }, + "storage": { + "median": 46172357, + "min": 46172357, + "max": 46172357 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 360.6637888889659, + "min": 360.6637888889659, + "max": 360.6637888889659 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 4.730110110390228, + "min": 4.730110110390228, + "max": 4.730110110390228 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "dispatches": [ + { + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "canary-screening-02", + "sample": "1" + }, + "ref": "run-tool-family-cases", + "state": "requested", + "requested_at": "2026-10-05T10:22:52Z", + "run_id": 37296210599, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296210599" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296210599", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296210599", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0a75d624e02f00f92884e89ebc0ed3201885a7ffa2e4cb898b238b26f5f10da9" + } + ], + "report_state": "current", + "report_path": "results/helix-nix/canary-screening-02/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37296210599, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-helix-nix-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-helix-nix-2026-10-05/benchmarks-37296210599.tar.gz", + "bundle_bytes": 1840218, + "sha256": "a888a59f9db1f52e986d17c2062d2ef363e69d5367861a735e65b61a99cc7ac1", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "direct-dispatch-qualification", + "plan_path": "results/hugo-go/direct-dispatch-qualification/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "bfad4beee177fef96c8b7c6eb3aee4df15d33fa69d5b44ba39b5fee5651f0d5b", + "definition_sha256": "c7923cefca6e69c5ade419e595ff7093be51a2df0cda6e14c6c79b7ff6d6b696", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 79.5, + "min": 78, + "max": 81 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 17.0, + "min": 11, + "max": 23 + }, + "storage": { + "median": 161851116.5, + "min": 161849774, + "max": 161852459 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 76.0, + "min": 72, + "max": 80 + }, + "storage": { + "median": 646497724.5, + "min": 604312219, + "max": 688683230 + }, + "storage_measured_count": 2 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 18.0, + "min": 16, + "max": 20 + }, + "storage": { + "median": 811503108.0, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 2 + } + ], + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "direct-dispatch-qualification", + "sample": "1" + }, + "ref": "benchmark-consolidation-2026-10-02", + "state": "requested", + "requested_at": "2026-10-02T17:03:43Z", + "run_id": 37038153301, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038153301" + }, + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "direct-dispatch-qualification", + "sample": "2" + }, + "ref": "benchmark-consolidation-2026-10-02", + "state": "requested", + "requested_at": "2026-10-02T17:03:46Z", + "run_id": 37038157657, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038157657" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "direct-dispatch-qualification", + "sample": 1, + "run_id": "37038153301", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038153301", + "source_sha": "aef10ff570e92395652bc8c5d7a4fe7cfd0bd114", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "9fff7441d035ae732e83c862aaa47ac64bbc7b429e840a56a864b21140239510" + }, + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "direct-dispatch-qualification", + "sample": 2, + "run_id": "37038157657", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37038157657", + "source_sha": "aef10ff570e92395652bc8c5d7a4fe7cfd0bd114", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "65d03d9f9b79f124339cf07c94d7a6cb63a8e5e770e8d84cc9f4d0a5f1fc7cc8" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/direct-dispatch-qualification/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37038153301, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's attempts, jobs, logs, artifacts, commit metadata, and workflow; not the original repository history", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-direct-dispatch-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-direct-dispatch-2026-10-02/benchmarks-37038153301.tar", + "bundle_bytes": 303616, + "sha256": "0780c3e940a26b78f835bf094a01f9d65c2490e6d52c7c08dbb6c10ce686e601", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37038157657, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's attempts, jobs, logs, artifacts, commit metadata, and workflow; not the original repository history", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-direct-dispatch-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-direct-dispatch-2026-10-02/benchmarks-37038157657.tar", + "bundle_bytes": 303616, + "sha256": "66debe7af8c10f0f9ec657aa58d9048029180143dc02eaa0d0d279138bc6b674", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "independent-samples-01", + "plan_path": "results/hugo-go/independent-samples-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "fc20058701dd1bf0b6a94acf40bf1d76adda8e37695472b295ec9bcc4b4da562", + "definition_sha256": "21702e83473c3a7a99cae19b7b4aa108243f49e8064878e4632804d315988a52", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 70.0, + "min": 61, + "max": 79 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 9.0, + "min": 9, + "max": 9 + }, + "storage": { + "median": 161808547.0, + "min": 161804322, + "max": 161812772 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 87.5, + "min": 85, + "max": 90 + }, + "storage": { + "median": 612837451.5, + "min": 610123303, + "max": 615551600 + }, + "storage_measured_count": 2 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 21.0, + "min": 18, + "max": 24 + }, + "storage": { + "median": 811503108.0, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 2 + } + ], + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "series_id": "independent-samples-01", + "sample": "1" + }, + "ref": "independent-benchmark-samples", + "state": "requested", + "requested_at": "2026-10-05T09:10:00Z", + "run_id": 37288240708, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288240708" + }, + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "series_id": "independent-samples-01", + "sample": "2" + }, + "ref": "independent-benchmark-samples", + "state": "requested", + "requested_at": "2026-10-05T09:10:02Z", + "run_id": 37288246103, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288246103" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "independent-samples-01", + "sample": 1, + "run_id": "37288240708", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288240708", + "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4fd9ad410330fc0e34949e2b298d079f188b543286ed7c662e52bcb969d5c925" + }, + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "independent-samples-01", + "sample": 2, + "run_id": "37288246103", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37288246103", + "source_sha": "874ab9890b328d8583f0b67e46a02f72393cf015", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "67db601ec3dfd03345ef6540efe4716d76d8398fb6b1d4c1f6dfbfa8737c4909" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/independent-samples-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37288240708, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "One declared Hugo Go sample, including its attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288240708.tar.gz", + "bundle_bytes": 192719, + "sha256": "5d1eba3f14c045aebe3dd0a8926ca435d8ed5a994794f8050e121a2917621ef3", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37288246103, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "One declared Hugo Go sample, including its attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-independent-samples-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-independent-samples-2026-10-05/benchmarks-37288246103.tar.gz", + "bundle_bytes": 192849, + "sha256": "9cbbc3f43c428fec36a3d6c642826156335170204c098af683034b5168d7d67a", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "merged-main-qualification-03", + "plan_path": "results/hugo-go/merged-main-qualification-03/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "fb020c5ffe80dad97e564738deb8f4192835b3551b3ab025d42c2103ada169c5", + "definition_sha256": "8575b4c7e5c1a048ccb9a3a36f7653474e78b0aea76f6166818bd51560409a39", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 53, + "min": 53, + "max": 53 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 10, + "min": 10, + "max": 10 + }, + "storage": { + "median": 161848750, + "min": 161848750, + "max": 161848750 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 69, + "min": 69, + "max": 69 + }, + "storage": { + "median": 588257602, + "min": 588257602, + "max": 588257602 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 16, + "min": 16, + "max": 16 + }, + "storage": { + "median": 811503108, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "merged-main-qualification-03", + "sample": "1" + }, + "ref": "main", + "state": "requested", + "requested_at": "2026-10-04T17:02:40Z", + "run_id": 37219016891, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219016891" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "merged-main-qualification-03", + "sample": 1, + "run_id": "37219016891", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219016891", + "source_sha": "fac4677ae1b217db78662291b34fa89d6253f92c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d7c4ede0806a1b7450b24d025ffa50ff9baec7de1d7e9adf12fd34c1d783a413" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/merged-main-qualification-03/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37219016891, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-merged-harness-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-merged-harness-2026-10-04/benchmarks-37219016891.tar.gz", + "bundle_bytes": 202782, + "sha256": "33a7ad7853c1ea64a9a923dc74389a5dd850fc6197af5dfa59bc0091891bef06", + "publication_verification": "Downloaded published assets, compared archive digests, extracted each bundle, and verified scoped file inventory and completeness" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "shared-evidence-01", + "plan_path": "results/hugo-go/shared-evidence-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "5885c0cb1808992f61acfcdcb136173b0820f554c6d9f0417b10a97f24b1bbb4", + "definition_sha256": "21e92460ce19eeb2e7a57492824a3729ca2e34df345f66e94a0a905df1410e70", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 52, + "min": 52, + "max": 52 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 8, + "min": 8, + "max": 8 + }, + "storage": { + "median": 161840391, + "min": 161840391, + "max": 161840391 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 98, + "min": 98, + "max": 98 + }, + "storage": { + "median": 588257602, + "min": 588257602, + "max": 588257602 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 17, + "min": 17, + "max": 17 + }, + "storage": { + "median": 811503108, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "series_id": "shared-evidence-01", + "sample": "1" + }, + "ref": "simplify-benchmark-cases", + "state": "requested", + "requested_at": "2026-10-05T10:03:05Z", + "run_id": 37294037661, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37294037661" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-evidence-01", + "sample": 1, + "run_id": "37294037661", "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37294037661", "source_sha": "aba77225af73e7064cd3ad3cbedd5fbac43217c3", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "806bba65ba732ca6135375b5021603315a6b6ae182614b9d37daa06d7cc9acbb" + "evidence_manifest_sha256": "806bba65ba732ca6135375b5021603315a6b6ae182614b9d37daa06d7cc9acbb" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/shared-evidence-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37294037661, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "One Hugo Go cold/warm sample, including attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-evidence-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-evidence-2026-10-05/benchmarks-37294037661.tar.gz", + "bundle_bytes": 184087, + "sha256": "8c73e249492a866a38764b6f155bdf9d31c32b41f7dd7a41d8210b29efd66441", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "shared-hugo-go", + "plan_path": "results/hugo-go/shared-hugo-go/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "ccd52e95b2380ca3e05635f9e982a9be9df0a51913192469eddf13c9f713f06d", + "definition_sha256": "7b55837951e12fd276a7141fabbed970820b13bc81eefa94ada5520aa38020fc", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 71.0, + "min": 63, + "max": 79 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 9.5, + "min": 9, + "max": 10 + }, + "storage": { + "median": 161823175.0, + "min": 161810724, + "max": 161835626 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 93.0, + "min": 91, + "max": 95 + }, + "storage": { + "median": 732124253.5, + "min": 652745399, + "max": 811503108 + }, + "storage_measured_count": 2 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 27.0, + "min": 20, + "max": 34 + }, + "storage": { + "median": 732124253.5, + "min": 652745399, + "max": 811503108 + }, + "storage_measured_count": 2 + } + ], + "dispatches": [], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-hugo-go", + "sample": 1, + "run_id": "37017469085", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37017469085", + "source_sha": "e6a4030e908cff132c942db574a8819597caccea", + "github_conclusion": "success", + "status": "failed", + "errors": [ + "2026-10-02T14:08:59.5369049Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:08:59.5369023Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:07:51.0368756Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:07:51.0368713Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss" + ], + "evidence_manifest_sha256": "a312eaccab1d4ade29a08b60d56838f2c112212a35e85e1280dadab188790f72" + }, + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-hugo-go", + "sample": 2, + "run_id": "37017742296", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37017742296", + "source_sha": "e6a4030e908cff132c942db574a8819597caccea", + "github_conclusion": "success", + "status": "failed", + "errors": [ + "2026-10-02T14:12:29.9176397Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:12:29.9176349Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:11:31.8617554Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", + "2026-10-02T14:11:31.8617514Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss" + ], + "evidence_manifest_sha256": "043a3a2adf6766392284d8cbdac77509b1b4c28be4f3ca735cb1b01d25019233" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/shared-hugo-go/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37017469085, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37017469085.tar", + "bundle_bytes": 320512, + "sha256": "7b7e11bc7146519204c9be9d1b245920c92ade858bb9909ddce8eaed7a1ad025", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37017742296, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37017742296.tar", + "bundle_bytes": 320000, + "sha256": "67fea908bc361a9ad2ef39632949d15da4206d5b2685077e603189ae66ae8f18", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "plan_path": "results/hugo-go/shared-lifecycle-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312", + "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 84, + "min": 84, + "max": 84 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 8, + "min": 8, + "max": 8 + }, + "storage": { + "median": 161784439, + "min": 161784439, + "max": 161784439 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 63, + "min": 63, + "max": 63 + }, + "storage": { + "median": 608031575, + "min": 608031575, + "max": 608031575 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 15, + "min": 15, + "max": 15 + }, + "storage": { + "median": 811503108, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo-go", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo-go", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:08:54Z", + "run_id": 37204582423, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204582423", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3154a932609e31cbdab09aa47b7e0671d9563ad8f438ae2a9b8b1c218c17a3d5" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/shared-lifecycle-qualification-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204582423, + "inventory_state": "complete", + "verified_files": 17, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204582423.tar", + "bundle_bytes": 335872, + "sha256": "f995093328e6357a396811765a14993f43011983d20190c2b91dd0d9518e5368", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo-go", + "series_id": "wrapper-qualification", + "plan_path": "results/hugo-go/wrapper-qualification/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "bcea77069750d79a24b92befbd1623ebf01c5b4a96a734ceecf0aa5ae2f4f810", + "definition_sha256": "6b17da60f8d8903040a8e63f02abc288bc44a4fdfd6ff501a0a838f3975a3e8d", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 85.0, + "min": 81, + "max": 89 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 2, + "measurement": { + "median": 11.5, + "min": 8, + "max": 15 + }, + "storage": { + "median": 161812028.0, + "min": 161785542, + "max": 161838514 + }, + "storage_measured_count": 2 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 92.5, + "min": 90, + "max": 95 + }, + "storage": { + "median": 641687960.0, + "min": 608031575, + "max": 675344345 + }, + "storage_measured_count": 2 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": { + "median": 20.0, + "min": 18, + "max": 22 + }, + "storage": { + "median": 811503108.0, + "min": 811503108, + "max": 811503108 + }, + "storage_measured_count": 2 + } + ], + "dispatches": [], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "wrapper-qualification", + "sample": 1, + "run_id": "37021492460", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37021492460", + "source_sha": "2329a034fea00236488cfc4aa73a281b4807bd2c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "042ce2d393e2cee1a99df5f1b71f17efcdbb2f51371ccc84531269bf38133c7e" + }, + { + "schema_version": 1, + "case_id": "hugo-go", + "series_id": "wrapper-qualification", + "sample": 2, + "run_id": "37021496989", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37021496989", + "source_sha": "2329a034fea00236488cfc4aa73a281b4807bd2c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "13327e14d4d5d7b805104a5f3162dced479764626f9291c1a2fec490d4637bae" + } + ], + "report_state": "current", + "report_path": "results/hugo-go/wrapper-qualification/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37021492460, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37021492460.tar", + "bundle_bytes": 450560, + "sha256": "3c9582d96484f984d6ba1153fff4bd0d94ae4ffc69e4233243c2d53d2e4d8ca2", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37021496989, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37021496989.tar", + "bundle_bytes": 450560, + "sha256": "7e2598fd4538ab15f4f099677d23f292ffeac682734a037b2e4a4f8def20d61d", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "plan_path": "results/hugo/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", + "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:50Z", + "run_id": 37104595025, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-01", + "sample": 1, + "run_id": "37104595025", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" + } + ], + "report_state": "current", + "report_path": "results/hugo/image-output-qualification-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37104595025, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104595025.tar", + "bundle_bytes": 235520, + "sha256": "af483c8478fcd8b726aff693fd2cea0281e91030e70a9af77218c6c3ad342f01", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "plan_path": "results/hugo/image-output-qualification-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", + "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 270, + "min": 270, + "max": 270 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 223, + "min": 223, + "max": 223 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 28, + "min": 28, + "max": 28 + }, + "storage": { + "median": 411514484, + "min": 411514484, + "max": 411514484 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:34:23Z", + "run_id": 37148378881, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148378881", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6560d3ffaa57bd746e57c74503987869c470c24ad4b4a02632b6392ad18239aa" + } + ], + "report_state": "current", + "report_path": "results/hugo/image-output-qualification-02/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37148378881, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148378881.tar", + "bundle_bytes": 263168, + "sha256": "de853e9d1c1b0b6bead7e1822a5efbc3a0bb92b15e37e1e53c400757b2354c56", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "merged-main-qualification-03", + "plan_path": "results/hugo/merged-main-qualification-03/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "09be21445afddc73425c579f525bf67ebdedf7a1fba0047db1f97b68f2a5f578", + "definition_sha256": "8522975a5e5fe5e8671da5ad5777b878e1b030e127caa4053e27483667e0bed7", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 211, + "min": 211, + "max": 211 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 22, + "min": 22, + "max": 22 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 222, + "min": 222, + "max": 222 + }, + "storage": { + "median": 411516403, + "min": 411516403, + "max": 411516403 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 25, + "min": 25, + "max": 25 + }, + "storage": { + "median": 411516403, + "min": 411516403, + "max": 411516403 + }, + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "merged-main-qualification-03", + "sample": "1" + }, + "ref": "main", + "state": "requested", + "requested_at": "2026-10-04T17:02:48Z", + "run_id": 37219024516, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219024516" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "merged-main-qualification-03", + "sample": 1, + "run_id": "37219024516", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219024516", + "source_sha": "fac4677ae1b217db78662291b34fa89d6253f92c", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "a7c622eb9e7b6ed0162f9d8f609bfab1660e0cc89d0b533a6c28132831969b20" + } + ], + "report_state": "current", + "report_path": "results/hugo/merged-main-qualification-03/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37219024516, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-merged-harness-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-merged-harness-2026-10-04/benchmarks-37219024516.tar.gz", + "bundle_bytes": 274236, + "sha256": "54c5c9372f50d6e94a73123cbff792adba3901a8862d8588c8c71ffc0d93ea51", + "publication_verification": "Downloaded published assets, compared archive digests, extracted each bundle, and verified scoped file inventory and completeness" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "migration-screening-01", + "plan_path": "results/hugo/migration-screening-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "4b17a1c1f142037c027f64a34b25a6976147117df83bf917f761acad3167f0d4", + "definition_sha256": "d08489cbef3d0c4d6a2e20d5f87fd3c824b1f6cb82c9d942c603df0be750af82", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "migration-screening-01", + "sample": "1" + }, + "ref": "active-case-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T01:14:38Z", + "run_id": 37085337504, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085337504" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "migration-screening-01", + "sample": 1, + "run_id": "37085337504", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085337504", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled" + ], + "evidence_manifest_sha256": "67f63d8d99d4228bffae1d8e5712af01708f0deaaac864e84b427e7d06b9e27d" } ], "report_state": "current", - "report_path": "results/hugo-go/shared-evidence-01/report.json", + "report_path": "results/hugo/migration-screening-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37294037661, + "run_id": 37085337504, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 7, "gaps": [], - "scope": "One Hugo Go cold/warm sample, including attempts, jobs, logs, artifacts, commit and workflow", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-evidence-2026-10-05", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-evidence-2026-10-05/benchmarks-37294037661.tar.gz", - "bundle_bytes": 184087, - "sha256": "8c73e249492a866a38764b6f155bdf9d31c32b41f7dd7a41d8210b29efd66441", - "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085337504.tar", + "bundle_bytes": 87040, + "sha256": "a0e0ada74707cf049f32c6efb77d9a2aa9e983e1d8a9eaa79a4ff7b414ad973c", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "shared-hugo-go", - "plan_path": "results/hugo-go/shared-hugo-go/series.json", + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "plan_path": "results/hugo/shared-lifecycle-qualification-01/series.json", "lane": "fresh", "variant": null, - "sample_count": 2, + "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", "providers": [ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", @@ -3535,23 +5789,139 @@ } ] }, - "plan_sha256": "ccd52e95b2380ca3e05635f9e982a9be9df0a51913192469eddf13c9f713f06d", - "definition_sha256": "7b55837951e12fd276a7141fabbed970820b13bc81eefa94ada5520aa38020fc", + "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4", + "definition_sha256": "8efc5a2368ac2012b1d1264006447fe93436d593e309aa867826b42a2d127750", "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-01", + "sample": "1" + }, + "ref": "shared-lifecycle-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:08:56Z", + "run_id": 37204584632, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-01", + "sample": 1, + "run_id": "37204584632", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache hugo cold: failure", + "GitHub Actions hugo cold: failure" + ], + "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" + } + ], + "report_state": "current", + "report_path": "results/hugo/shared-lifecycle-qualification-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37204584632, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204584632.tar", + "bundle_bytes": 147968, + "sha256": "0b21b4ea8ef41cc7fc8f14200aaa13a58e398da102d4a983f5673f64d3091544", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", + "plan_path": "results/hugo/shared-lifecycle-qualification-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "gohugoio/hugo", + "pins": [ + { + "path": "upstream", + "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2", + "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": true, + "execution_verified": true, + "methodology_issues": [], "missing_observations": [], "summaries": [ { "phase": "cold", "provider": "actions-cache", - "count": 2, + "count": 1, "measurement": { - "median": 71.0, - "min": 63, - "max": 79 + "median": 229, + "min": 229, + "max": 229 }, "storage": null, "storage_measured_count": 0 @@ -3559,127 +5929,105 @@ { "phase": "warm", "provider": "actions-cache", - "count": 2, + "count": 1, "measurement": { - "median": 9.5, - "min": 9, - "max": 10 - }, - "storage": { - "median": 161823175.0, - "min": 161810724, - "max": 161835626 + "median": 17, + "min": 17, + "max": 17 }, - "storage_measured_count": 2 + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", - "count": 2, + "count": 1, "measurement": { - "median": 93.0, - "min": 91, - "max": 95 + "median": 226, + "min": 226, + "max": 226 }, "storage": { - "median": 732124253.5, - "min": 652745399, - "max": 811503108 + "median": 411517974, + "min": 411517974, + "max": 411517974 }, - "storage_measured_count": 2 + "storage_measured_count": 1 }, { "phase": "warm", "provider": "boringcache", - "count": 2, + "count": 1, "measurement": { - "median": 27.0, - "min": 20, - "max": 34 + "median": 27, + "min": 27, + "max": 27 }, "storage": { - "median": 732124253.5, - "min": 652745399, - "max": 811503108 + "median": 411517974, + "min": 411517974, + "max": 411517974 }, - "storage_measured_count": 2 + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "hugo", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "hugo", + "cli_version": "v1.33.0", + "series_id": "shared-lifecycle-qualification-02", + "sample": "1" + }, + "ref": "shared-lifecycle-correction-2026-10-04", + "state": "requested", + "requested_at": "2026-10-04T13:17:46Z", + "run_id": 37205110890, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890" } ], - "dispatches": [], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "shared-hugo-go", + "case_id": "hugo", + "series_id": "shared-lifecycle-qualification-02", "sample": 1, - "run_id": "37017469085", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37017469085", - "source_sha": "e6a4030e908cff132c942db574a8819597caccea", - "github_conclusion": "success", - "status": "failed", - "errors": [ - "2026-10-02T14:08:59.5369049Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:08:59.5369023Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:07:51.0368756Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:07:51.0368713Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss" - ], - "evidence_manifest_sha256": "a312eaccab1d4ade29a08b60d56838f2c112212a35e85e1280dadab188790f72" - }, - { - "schema_version": 1, - "case_id": "hugo-go", - "series_id": "shared-hugo-go", - "sample": 2, - "run_id": "37017742296", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37017742296", - "source_sha": "e6a4030e908cff132c942db574a8819597caccea", + "run_id": "37205110890", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", "github_conclusion": "success", - "status": "failed", - "errors": [ - "2026-10-02T14:12:29.9176397Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:12:29.9176349Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:11:31.8617554Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss", - "2026-10-02T14:11:31.8617514Z ##[warning]boringcache/one save failed: Input does not meet YAML 1.2 \"Core Schema\" specification: fail-on-cache-miss" - ], - "evidence_manifest_sha256": "043a3a2adf6766392284d8cbdac77509b1b4c28be4f3ca735cb1b01d25019233" + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6f45e249bf9659845af18a99f80367c298095a20957e661418f78bba9b15c2ac" } ], "report_state": "current", - "report_path": "results/hugo-go/shared-hugo-go/report.json", + "report_path": "results/hugo/shared-lifecycle-qualification-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37017469085, - "inventory_state": "complete", - "verified_files": 15, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37017469085.tar", - "bundle_bytes": 320512, - "sha256": "7b7e11bc7146519204c9be9d1b245920c92ade858bb9909ddce8eaed7a1ad025", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37017742296, + "run_id": 37205110890, "inventory_state": "complete", "verified_files": 15, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37017742296.tar", - "bundle_bytes": 320000, - "sha256": "67fea908bc361a9ad2ef39632949d15da4206d5b2685077e603189ae66ae8f18", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205110890.tar", + "bundle_bytes": 517120, + "sha256": "d199516ad802b33ea4d5789998a4693c224a3c992f7e59750536e4e6358cc472", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "shared-lifecycle-qualification-01", - "plan_path": "results/hugo-go/shared-lifecycle-qualification-01/series.json", + "case_id": "immich", + "series_id": "image-output-qualification-01", + "plan_path": "results/immich/image-output-qualification-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, @@ -3689,29 +6037,32 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "immich-app/immich", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", "kind": "gitlink" } ] }, - "plan_sha256": "35a2dc05b7b87a66a9da5682af58e51e412316e72c350c17078ed04080c4a312", - "definition_sha256": "908a8962c0bcdca6e1d3842c2e64bfe3ba71735a16821012d5767acd775693cb", + "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", + "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -3719,9 +6070,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 84, - "min": 84, - "max": 84 + "median": 371, + "min": 371, + "max": 371 }, "storage": null, "storage_measured_count": 0 @@ -3731,30 +6082,26 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 8, - "min": 8, - "max": 8 - }, - "storage": { - "median": 161784439, - "min": 161784439, - "max": 161784439 + "median": 57, + "min": 57, + "max": 57 }, - "storage_measured_count": 1 + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", "count": 1, "measurement": { - "median": 63, - "min": 63, - "max": 63 + "median": 207, + "min": 207, + "max": 207 }, "storage": { - "median": 608031575, - "min": 608031575, - "max": 608031575 + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 }, "storage_measured_count": 1 }, @@ -3763,117 +6110,120 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 15, - "min": 15, - "max": 15 + "median": 38, + "min": 38, + "max": 38 }, "storage": { - "median": 811503108, - "min": 811503108, - "max": 811503108 + "median": 1758920230, + "min": 1758920230, + "max": 1758920230 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "hugo-go", + "case_id": "immich", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo-go", + "case_id": "immich", "cli_version": "v1.33.0", - "series_id": "shared-lifecycle-qualification-01", + "series_id": "image-output-qualification-01", "sample": "1" }, - "ref": "shared-lifecycle-2026-10-04", + "ref": "image-output-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-04T13:08:54Z", - "run_id": 37204582423, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423" + "requested_at": "2026-10-03T06:54:59Z", + "run_id": 37104602471, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "shared-lifecycle-qualification-01", + "case_id": "immich", + "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37204582423", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204582423", - "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "run_id": "37104602471", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "3154a932609e31cbdab09aa47b7e0671d9563ad8f438ae2a9b8b1c218c17a3d5" + "evidence_manifest_sha256": "a232a9c86cf53c4945b672566ad89e76085267e61bdd5d7b0226606108e41521" } ], "report_state": "current", - "report_path": "results/hugo-go/shared-lifecycle-qualification-01/report.json", + "report_path": "results/immich/image-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37204582423, + "run_id": 37104602471, "inventory_state": "complete", - "verified_files": 17, + "verified_files": 13, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204582423.tar", - "bundle_bytes": 335872, - "sha256": "f995093328e6357a396811765a14993f43011983d20190c2b91dd0d9518e5368", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104602471.tar", + "bundle_bytes": 416256, + "sha256": "89dd58ddf791fbd4b2d66280cca759c2cff6f2f6bdbc4498ffeda57ed9185c87", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo-go", - "series_id": "wrapper-qualification", - "plan_path": "results/hugo-go/wrapper-qualification/series.json", + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", + "plan_path": "results/linkerd2/image-output-qualification-01/series.json", "lane": "fresh", "variant": null, - "sample_count": 2, + "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", "providers": [ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "linkerd/linkerd2", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", "kind": "gitlink" } ] }, - "plan_sha256": "bcea77069750d79a24b92befbd1623ebf01c5b4a96a734ceecf0aa5ae2f4f810", - "definition_sha256": "6b17da60f8d8903040a8e63f02abc288bc44a4fdfd6ff501a0a838f3975a3e8d", + "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", + "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { "phase": "cold", "provider": "actions-cache", - "count": 2, + "count": 1, "measurement": { - "median": 85.0, - "min": 81, - "max": 89 + "median": 306, + "min": 306, + "max": 306 }, "storage": null, "storage_measured_count": 0 @@ -3881,117 +6231,105 @@ { "phase": "warm", "provider": "actions-cache", - "count": 2, + "count": 1, "measurement": { - "median": 11.5, - "min": 8, + "median": 15, + "min": 15, "max": 15 }, - "storage": { - "median": 161812028.0, - "min": 161785542, - "max": 161838514 - }, - "storage_measured_count": 2 + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", - "count": 2, + "count": 1, "measurement": { - "median": 92.5, - "min": 90, - "max": 95 + "median": 200, + "min": 200, + "max": 200 }, "storage": { - "median": 641687960.0, - "min": 608031575, - "max": 675344345 + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 }, - "storage_measured_count": 2 + "storage_measured_count": 1 }, { "phase": "warm", "provider": "boringcache", - "count": 2, + "count": 1, "measurement": { - "median": 20.0, - "min": 18, - "max": 22 + "median": 12, + "min": 12, + "max": 12 }, "storage": { - "median": 811503108.0, - "min": 811503108, - "max": 811503108 + "median": 1230761384, + "min": 1230761384, + "max": 1230761384 }, - "storage_measured_count": 2 + "storage_measured_count": 1 + } + ], + "dispatches": [ + { + "case_id": "linkerd2", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0", + "series_id": "image-output-qualification-01", + "sample": "1" + }, + "ref": "image-output-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T06:54:52Z", + "run_id": 37104596700, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700" } ], - "dispatches": [], "completions": [ { "schema_version": 1, - "case_id": "hugo-go", - "series_id": "wrapper-qualification", + "case_id": "linkerd2", + "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37021492460", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37021492460", - "source_sha": "2329a034fea00236488cfc4aa73a281b4807bd2c", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "042ce2d393e2cee1a99df5f1b71f17efcdbb2f51371ccc84531269bf38133c7e" - }, - { - "schema_version": 1, - "case_id": "hugo-go", - "series_id": "wrapper-qualification", - "sample": 2, - "run_id": "37021496989", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37021496989", - "source_sha": "2329a034fea00236488cfc4aa73a281b4807bd2c", + "run_id": "37104596700", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "13327e14d4d5d7b805104a5f3162dced479764626f9291c1a2fec490d4637bae" + "evidence_manifest_sha256": "3eba28538f46405b75d696f6c8071d56e26bf96e6b1be1f7fc5479893e282adf" } ], "report_state": "current", - "report_path": "results/hugo-go/wrapper-qualification/report.json", + "report_path": "results/linkerd2/image-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37021492460, - "inventory_state": "complete", - "verified_files": 15, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37021492460.tar", - "bundle_bytes": 450560, - "sha256": "3c9582d96484f984d6ba1153fff4bd0d94ae4ffc69e4233243c2d53d2e4d8ca2", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37021496989, + "run_id": 37104596700, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 13, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-wrapper-qualification-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-wrapper-qualification-2026-10-02/benchmarks-37021496989.tar", - "bundle_bytes": 450560, - "sha256": "7e2598fd4538ab15f4f099677d23f292ffeac682734a037b2e4a4f8def20d61d", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104596700.tar", + "bundle_bytes": 354816, + "sha256": "63efd14ac9e14ffd6b09b3ec57786f72aba962deda6f19fb773296b7428e1a60", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "image-output-qualification-01", - "plan_path": "results/hugo/image-output-qualification-01/series.json", + "case_id": "linkerd2", + "series_id": "migration-screening-01", + "plan_path": "results/linkerd2/migration-screening-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, @@ -4001,24 +6339,24 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "linkerd/linkerd2", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", "kind": "gitlink" } ] }, - "plan_sha256": "cfa4c56832b46420bd7ddc24e90cd7a7d364f487a7498ca29658395a356c7700", - "definition_sha256": "200bdce61b6cbba1361be02b435abbe9c4750e77dd72058281ba1c1e8a8882f4", + "plan_sha256": "8dc741e2abd94953818d2db8a9f0d4648b7ca290e1f1972e87be1fa0c9eb3cd5", + "definition_sha256": "9458671ff40345d67431c0b0cdb14a6d4e7c6ca38a2a40766e11eb85ee3eae02", "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, @@ -4046,70 +6384,220 @@ "actions-cache" ] ], - "summaries": [], + "summaries": [], + "dispatches": [ + { + "case_id": "linkerd2", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "linkerd2", + "cli_version": "v1.33.0", + "series_id": "migration-screening-01", + "sample": "1" + }, + "ref": "active-case-qualification-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T01:14:40Z", + "run_id": 37085339156, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085339156" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "linkerd2", + "series_id": "migration-screening-01", + "sample": 1, + "run_id": "37085339156", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085339156", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "cancelled", + "status": "failed", + "errors": [ + "Workflow concluded cancelled" + ], + "evidence_manifest_sha256": "91fd2da3d5203e0dac84d707beb7774c7689536058904f21baf6f27879260f68" + } + ], + "report_state": "current", + "report_path": "results/linkerd2/migration-screening-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37085339156, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085339156.tar", + "bundle_bytes": 87040, + "sha256": "f7d828874085433b11447c700c224bb24d3f7237a290a9cfdb31ef0ef4d52880", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "mastodon", + "series_id": "image-output-qualification-01", + "plan_path": "results/mastodon/image-output-qualification-01/series.json", + "lane": "fresh", + "variant": "server", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "mastodon/mastodon", + "pins": [ + { + "path": "upstream", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", + "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 736, + "min": 736, + "max": 736 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 59, + "min": 59, + "max": 59 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 424, + "min": 424, + "max": 424 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 38, + "min": 38, + "max": 38 + }, + "storage": { + "median": 1130445165, + "min": 1130445165, + "max": 1130445165 + }, + "storage_measured_count": 1 + } + ], "dispatches": [ { - "case_id": "hugo", + "case_id": "mastodon", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", + "case_id": "mastodon", "cli_version": "v1.33.0", + "variant": "server", "series_id": "image-output-qualification-01", "sample": "1" }, "ref": "image-output-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T06:54:50Z", - "run_id": 37104595025, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025" + "requested_at": "2026-10-03T06:54:55Z", + "run_id": 37104599574, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", + "case_id": "mastodon", "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37104595025", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104595025", + "run_id": "37104599574", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "BoringCache hugo cold: failure", - "GitHub Actions hugo cold: failure" - ], - "evidence_manifest_sha256": "12aebc138373729492fa362c157080777851485d91d328d4d7a0209618719ce5" + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2e2e3c52be0fbdb8c2ea7924416d0bc1530b9ee9b391a13b8f3fc03b0ab61e52" } ], "report_state": "current", - "report_path": "results/hugo/image-output-qualification-01/report.json", + "report_path": "results/mastodon/image-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104595025, + "run_id": 37104599574, "inventory_state": "complete", - "verified_files": 7, + "verified_files": 13, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104595025.tar", - "bundle_bytes": 235520, - "sha256": "af483c8478fcd8b726aff693fd2cea0281e91030e70a9af77218c6c3ad342f01", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104599574.tar", + "bundle_bytes": 694784, + "sha256": "8b6a772ff39503d0e9aac23a96e13f627e9ffbba1f4e84ece45598857559ee60", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "image-output-qualification-02", - "plan_path": "results/hugo/image-output-qualification-02/series.json", + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", + "plan_path": "results/mastodon/streaming-output-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "streaming", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -4124,17 +6612,17 @@ "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "mastodon/mastodon", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", "kind": "gitlink" } ] }, - "plan_sha256": "6b9cbe6a1abe5039dfa929d5fa898b3316a3fa019123db5473d97ec461aba485", - "definition_sha256": "d93cf392d1d565e2d563f3e5e7b947f28616e7c830f3a3c3a84fa10f4033ccb0", + "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", + "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, @@ -4150,9 +6638,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 270, - "min": 270, - "max": 270 + "median": 52, + "min": 52, + "max": 52 }, "storage": null, "storage_measured_count": 0 @@ -4162,9 +6650,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 28, - "min": 28, - "max": 28 + "median": 21, + "min": 21, + "max": 21 }, "storage": null, "storage_measured_count": 0 @@ -4174,14 +6662,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 223, - "min": 223, - "max": 223 + "median": 40, + "min": 40, + "max": 40 }, "storage": { - "median": 411514484, - "min": 411514484, - "max": 411514484 + "median": 106281953, + "min": 106281953, + "max": 106281953 }, "storage_measured_count": 1 }, @@ -4190,255 +6678,353 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 28, - "min": 28, - "max": 28 + "median": 22, + "min": 22, + "max": 22 }, "storage": { - "median": 411514484, - "min": 411514484, - "max": 411514484 + "median": 106281953, + "min": 106281953, + "max": 106281953 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "hugo", + "case_id": "mastodon", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", + "case_id": "mastodon", "cli_version": "v1.33.0", - "series_id": "image-output-qualification-02", + "variant": "streaming", + "series_id": "streaming-output-qualification-01", "sample": "1" }, - "ref": "docker-output-correction-2026-10-03", + "ref": "active-variant-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T19:34:23Z", - "run_id": 37148378881, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881" + "requested_at": "2026-10-03T19:45:29Z", + "run_id": 37149053291, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", - "series_id": "image-output-qualification-02", + "case_id": "mastodon", + "series_id": "streaming-output-qualification-01", "sample": 1, - "run_id": "37148378881", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148378881", - "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "run_id": "37149053291", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "6560d3ffaa57bd746e57c74503987869c470c24ad4b4a02632b6392ad18239aa" + "evidence_manifest_sha256": "530bf111ae6b4a6f70347e2e293b0ad75cec21dd175728be464cf8ecfc1ebfec" } ], "report_state": "current", - "report_path": "results/hugo/image-output-qualification-02/report.json", + "report_path": "results/mastodon/streaming-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37148378881, + "run_id": 37149053291, "inventory_state": "complete", "verified_files": 13, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148378881.tar", - "bundle_bytes": 263168, - "sha256": "de853e9d1c1b0b6bead7e1822a5efbc3a0bb92b15e37e1e53c400757b2354c56", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149053291.tar", + "bundle_bytes": 266240, + "sha256": "899c7adc2f77d143cfdda44d59b9f12884877c5a5fcf65bed1c5989341c8b2a9", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "merged-main-qualification-03", - "plan_path": "results/hugo/merged-main-qualification-03/series.json", + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "plan_path": "results/msgpack-sbt/registry-qualification-01/series.json", "lane": "fresh", "variant": null, - "sample_count": 1, + "sample_count": 2, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "gohugoio/hugo", + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", "pins": [ { - "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", - "kind": "gitlink" + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" } - ] + ], + "patch": "cache.patch" }, - "plan_sha256": "09be21445afddc73425c579f525bf67ebdedf7a1fba0047db1f97b68f2a5f578", - "definition_sha256": "8522975a5e5fe5e8671da5ad5777b878e1b030e127caa4053e27483667e0bed7", + "plan_sha256": "796afbd8d1867db7442db953d38cb7e35d757fb18ef0d8825213983de643110e", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 211, - "min": 211, - "max": 211 - }, + "provider": "bazel-remote", + "count": 2, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 22, - "min": 22, - "max": 22 - }, + "provider": "bazel-remote", + "count": 2, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", - "count": 1, - "measurement": { - "median": 222, - "min": 222, - "max": 222 - }, - "storage": { - "median": 411516403, - "min": 411516403, - "max": 411516403 - }, - "storage_measured_count": 1 + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", "provider": "boringcache", - "count": 1, - "measurement": { - "median": 25, - "min": 25, - "max": 25 - }, - "storage": { - "median": 411516403, - "min": 411516403, - "max": 411516403 - }, - "storage_measured_count": 1 + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "hugo", + "case_id": "msgpack-sbt", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", - "cli_version": "v1.33.0", - "series_id": "merged-main-qualification-03", + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", "sample": "1" }, - "ref": "main", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-04T17:02:48Z", - "run_id": 37219024516, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219024516" + "requested_at": "2026-10-05T11:24:22Z", + "run_id": 37302718511, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + }, + { + "case_id": "msgpack-sbt", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:24:24Z", + "run_id": 37302721578, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302721578" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", - "series_id": "merged-main-qualification-03", + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", "sample": 1, - "run_id": "37219024516", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37219024516", - "source_sha": "fac4677ae1b217db78662291b34fa89d6253f92c", + "run_id": "37302718511", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302718511", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f573580e4f886e0a21343f3377440101fa3dee54750ec852e996e3e5c926746c" + }, + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302721578", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302721578", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "a7c622eb9e7b6ed0162f9d8f609bfab1660e0cc89d0b533a6c28132831969b20" + "evidence_manifest_sha256": "23057a79736019f974fe24bd5d9448b6e0a2539f778f5f2e0ed1818c5806d449" } ], "report_state": "current", - "report_path": "results/hugo/merged-main-qualification-03/report.json", + "report_path": "results/msgpack-sbt/registry-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37219024516, + "run_id": 37302718511, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-merged-harness-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-merged-harness-2026-10-04/benchmarks-37219024516.tar.gz", - "bundle_bytes": 274236, - "sha256": "54c5c9372f50d6e94a73123cbff792adba3901a8862d8588c8c71ffc0d93ea51", - "publication_verification": "Downloaded published assets, compared archive digests, extracted each bundle, and verified scoped file inventory and completeness" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302718511.tar.gz", + "bundle_bytes": 1646194, + "sha256": "5cb9069086bdd60cd6064a68e4587b3ce20d9b62b14f3db266c8bbdeda51188c", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302721578, + "inventory_state": "complete", + "verified_files": 16, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302721578.tar.gz", + "bundle_bytes": 1648859, + "sha256": "77e5498f62a2ff1fe6bcb09728d423f5c60309e1c21e524941b4261d4ff733e7", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "migration-screening-01", - "plan_path": "results/hugo/migration-screening-01/series.json", + "case_id": "msgpack-sbt", + "series_id": "registry-screening-01", + "plan_path": "results/msgpack-sbt/registry-screening-01/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "gohugoio/hugo", + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", "pins": [ { - "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", - "kind": "gitlink" + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" } + ], + "patch": "cache.patch" + }, + "plan_sha256": "010d8fda4af1a1392392d646834c8bf43fd65102320ec4be80dc0ee2384a7581", + "definition_sha256": "4cfe1a5d85a53dbf87f068d5fe79baac4bbd97aa666fd5c6c0d78fa68434a89c", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "msgpack-sbt", + "series_id": "registry-screening-02", + "plan_path": "results/msgpack-sbt/registry-screening-02/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, - "plan_sha256": "4b17a1c1f142037c027f64a34b25a6976147117df83bf917f761acad3167f0d4", - "definition_sha256": "d08489cbef3d0c4d6a2e20d5f87fd3c824b1f6cb82c9d942c603df0be750af82", - "state": "failed", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "plan_sha256": "fdbff89944ec0f7576b6fcb214ff2d2329a53afbea7453e1edc10948d3f9599d", + "definition_sha256": "7838f47d75d6765b8840f66b06cc3a0496411c0794d19a966da28926704a2782", + "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ [ 1, @@ -4448,7 +7034,7 @@ [ 1, "cold", - "actions-cache" + "bazel-remote" ], [ 1, @@ -4458,71 +7044,151 @@ [ 1, "warm", - "actions-cache" + "bazel-remote" ] ], "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "msgpack-sbt", + "series_id": "registry-screening-03", + "plan_path": "results/msgpack-sbt/registry-screening-03/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "plan_sha256": "d599b25c1d0c358d407f5915f5c3a7648565b1d4e16ffd5ece73fb88cac437b4", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": true, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], "dispatches": [ { - "case_id": "hugo", + "case_id": "msgpack-sbt", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", - "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "case_id": "msgpack-sbt", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-03", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "main", "state": "requested", - "requested_at": "2026-10-03T01:14:38Z", - "run_id": 37085337504, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085337504" + "requested_at": "2026-10-05T11:17:39Z", + "run_id": 37302010045, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302010045" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", - "series_id": "migration-screening-01", + "case_id": "msgpack-sbt", + "series_id": "registry-screening-03", "sample": 1, - "run_id": "37085337504", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085337504", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", - "github_conclusion": "cancelled", - "status": "failed", - "errors": [ - "Workflow concluded cancelled" - ], - "evidence_manifest_sha256": "67f63d8d99d4228bffae1d8e5712af01708f0deaaac864e84b427e7d06b9e27d" + "run_id": "37302010045", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302010045", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "890ffdc934b57d911ca4cf05267c58040d3f0449a5144c011fbdcfe5c08f06e9" } ], "report_state": "current", - "report_path": "results/hugo/migration-screening-01/report.json", + "report_path": "results/msgpack-sbt/registry-screening-03/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085337504, + "run_id": 37302010045, "inventory_state": "complete", - "verified_files": 7, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085337504.tar", - "bundle_bytes": 87040, - "sha256": "a0e0ada74707cf049f32c6efb77d9a2aa9e983e1d8a9eaa79a4ff7b414ad973c", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302010045.tar.gz", + "bundle_bytes": 1659064, + "sha256": "1eaef318a5107d74bf9fa5109707ac16adb2a88ac0996b97961eebbe6b6c3ef5", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "shared-lifecycle-qualification-01", - "plan_path": "results/hugo/shared-lifecycle-qualification-01/series.json", + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", + "plan_path": "results/n8n/distroless-output-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "distroless", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -4530,115 +7196,151 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "n8n-io/n8n", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", "kind": "gitlink" } ] }, - "plan_sha256": "5a0c3c4b9e6cd65dc8a629f125534a1fcdd5ba9015d2f5b6aa3ba312145e53e4", - "definition_sha256": "8efc5a2368ac2012b1d1264006447fe93436d593e309aa867826b42a2d127750", - "state": "failed", + "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 214, + "min": 214, + "max": 214 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 132, + "min": 132, + "max": 132 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 81, + "min": 81, + "max": 81 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 110, + "min": 110, + "max": 110 + }, + "storage": { + "median": 581055738, + "min": 581055738, + "max": 581055738 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { - "case_id": "hugo", + "case_id": "n8n", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", + "case_id": "n8n", "cli_version": "v1.33.0", - "series_id": "shared-lifecycle-qualification-01", + "variant": "distroless", + "series_id": "distroless-output-qualification-01", "sample": "1" }, - "ref": "shared-lifecycle-2026-10-04", + "ref": "active-variant-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-04T13:08:56Z", - "run_id": 37204584632, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632" + "requested_at": "2026-10-03T19:45:33Z", + "run_id": 37149056843, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", - "series_id": "shared-lifecycle-qualification-01", + "case_id": "n8n", + "series_id": "distroless-output-qualification-01", "sample": 1, - "run_id": "37204584632", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204584632", - "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "BoringCache hugo cold: failure", - "GitHub Actions hugo cold: failure" - ], - "evidence_manifest_sha256": "18e637ca4b6502539b929702e3dacedee83940a51a1a2b75c82632c4dec2ad70" + "run_id": "37149056843", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "4bbc22e12b0a3e5e9c1c5c481f68f29002291e60a1bcc748ea3e18eaaed8566e" } ], "report_state": "current", - "report_path": "results/hugo/shared-lifecycle-qualification-01/report.json", + "report_path": "results/n8n/distroless-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37204584632, + "run_id": 37149056843, "inventory_state": "complete", - "verified_files": 7, + "verified_files": 13, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204584632.tar", - "bundle_bytes": 147968, - "sha256": "0b21b4ea8ef41cc7fc8f14200aaa13a58e398da102d4a983f5673f64d3091544", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149056843.tar", + "bundle_bytes": 1658880, + "sha256": "b1b97061460c375759d37184868d53cddad9c1b980b4e69d8e6a67cce374ce09", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "hugo", - "series_id": "shared-lifecycle-qualification-02", - "plan_path": "results/hugo/shared-lifecycle-qualification-02/series.json", + "case_id": "n8n", + "series_id": "image-output-qualification-01", + "plan_path": "results/n8n/image-output-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "docker", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -4646,29 +7348,32 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "gohugoio/hugo", + "repository": "n8n-io/n8n", "pins": [ { "path": "upstream", - "revision": "6b3ba3a7e22ae2809605b8118aeed08b244f1955", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", "kind": "gitlink" } ] }, - "plan_sha256": "047e1f8caac9070ad94eb62149d671a69d3a1214a5a917c374cc2ca3e43ae8f2", - "definition_sha256": "142454ba72794a7253d4db6b10e9209939361a495bb53094bae503fd830b044d", + "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", + "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -4676,9 +7381,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 229, - "min": 229, - "max": 229 + "median": 446, + "min": 446, + "max": 446 }, "storage": null, "storage_measured_count": 0 @@ -4688,9 +7393,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 17, - "min": 17, - "max": 17 + "median": 150, + "min": 150, + "max": 150 }, "storage": null, "storage_measured_count": 0 @@ -4700,14 +7405,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 226, - "min": 226, - "max": 226 + "median": 319, + "min": 319, + "max": 319 }, "storage": { - "median": 411517974, - "min": 411517974, - "max": 411517974 + "median": 483433384, + "min": 483433384, + "max": 483433384 }, "storage_measured_count": 1 }, @@ -4716,77 +7421,78 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 27, - "min": 27, - "max": 27 + "median": 162, + "min": 162, + "max": 162 }, "storage": { - "median": 411517974, - "min": 411517974, - "max": 411517974 + "median": 483433384, + "min": 483433384, + "max": 483433384 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "hugo", + "case_id": "n8n", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "hugo", + "case_id": "n8n", "cli_version": "v1.33.0", - "series_id": "shared-lifecycle-qualification-02", + "variant": "docker", + "series_id": "image-output-qualification-01", "sample": "1" }, - "ref": "shared-lifecycle-correction-2026-10-04", + "ref": "image-output-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-04T13:17:46Z", - "run_id": 37205110890, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890" + "requested_at": "2026-10-03T06:54:54Z", + "run_id": 37104598178, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178" } ], "completions": [ { "schema_version": 1, - "case_id": "hugo", - "series_id": "shared-lifecycle-qualification-02", + "case_id": "n8n", + "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37205110890", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205110890", - "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "run_id": "37104598178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "6f45e249bf9659845af18a99f80367c298095a20957e661418f78bba9b15c2ac" + "evidence_manifest_sha256": "5f0af1311b0f43733e4562504aed6f42dde1420f1c800abe59315d5f97a55214" } ], "report_state": "current", - "report_path": "results/hugo/shared-lifecycle-qualification-02/report.json", + "report_path": "results/n8n/image-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37205110890, + "run_id": 37104598178, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 13, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205110890.tar", - "bundle_bytes": 517120, - "sha256": "d199516ad802b33ea4d5789998a4693c224a3c992f7e59750536e4e6358cc472", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104598178.tar", + "bundle_bytes": 1748480, + "sha256": "0bb9fc327d8fef9a8af1eb8bdcf8ec699508a46715bae610a28c12b0fc1dd49f", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "immich", - "series_id": "image-output-qualification-01", - "plan_path": "results/immich/image-output-qualification-01/series.json", + "case_id": "n8n", + "series_id": "runners-output-qualification-01", + "plan_path": "results/n8n/runners-output-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "runners", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -4794,24 +7500,24 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "immich-app/immich", + "repository": "n8n-io/n8n", "pins": [ { "path": "upstream", - "revision": "007c18aec6cc9552fb7d8b7c791c41d7424876b0", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", "kind": "gitlink" } ] }, - "plan_sha256": "f2b98c3031e2c795b68c4eb5d0ad15b805c1bf4c8a70f015e2d0446ddc44a77f", - "definition_sha256": "aa7edd687de2be1ba9ea1393766e01359e5a0ac10d0ac59ec556f9c4d536eaab", + "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", + "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, @@ -4827,9 +7533,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 371, - "min": 371, - "max": 371 + "median": 144, + "min": 144, + "max": 144 }, "storage": null, "storage_measured_count": 0 @@ -4839,9 +7545,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 57, - "min": 57, - "max": 57 + "median": 89, + "min": 89, + "max": 89 }, "storage": null, "storage_measured_count": 0 @@ -4851,14 +7557,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 207, - "min": 207, - "max": 207 + "median": 62, + "min": 62, + "max": 62 }, "storage": { - "median": 1758920230, - "min": 1758920230, - "max": 1758920230 + "median": 413827357, + "min": 413827357, + "max": 413827357 }, "storage_measured_count": 1 }, @@ -4867,77 +7573,78 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 38, - "min": 38, - "max": 38 + "median": 60, + "min": 60, + "max": 60 }, "storage": { - "median": 1758920230, - "min": 1758920230, - "max": 1758920230 + "median": 413827357, + "min": 413827357, + "max": 413827357 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "immich", + "case_id": "n8n", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "immich", + "case_id": "n8n", "cli_version": "v1.33.0", - "series_id": "image-output-qualification-01", + "variant": "runners", + "series_id": "runners-output-qualification-01", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "active-variant-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T06:54:59Z", - "run_id": 37104602471, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471" + "requested_at": "2026-10-03T19:45:31Z", + "run_id": 37149055130, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" } ], "completions": [ { "schema_version": 1, - "case_id": "immich", - "series_id": "image-output-qualification-01", + "case_id": "n8n", + "series_id": "runners-output-qualification-01", "sample": 1, - "run_id": "37104602471", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104602471", - "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "run_id": "37149055130", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130", + "source_sha": "b759879316279afa06a7e8166c905429547adb34", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "a232a9c86cf53c4945b672566ad89e76085267e61bdd5d7b0226606108e41521" + "evidence_manifest_sha256": "c52e6d5bedd2e1d5e4333c820e0895b30eee8eb5c5ec02389cb5d669dc6f7017" } ], "report_state": "current", - "report_path": "results/immich/image-output-qualification-01/report.json", + "report_path": "results/n8n/runners-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104602471, + "run_id": 37149055130, "inventory_state": "complete", "verified_files": 13, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104602471.tar", - "bundle_bytes": 416256, - "sha256": "89dd58ddf791fbd4b2d66280cca759c2cff6f2f6bdbc4498ffeda57ed9185c87", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149055130.tar", + "bundle_bytes": 1602560, + "sha256": "36d2b8fe86fbfe42fa9df155468b0beda3b6a1baabec398dd3a577e04feafafd", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "linkerd2", - "series_id": "image-output-qualification-01", - "plan_path": "results/linkerd2/image-output-qualification-01/series.json", + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", + "plan_path": "results/n8n/shared-turbo-qualification-01/series.json", "lane": "fresh", - "variant": null, + "variant": "turbo", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -4945,32 +7652,29 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "linkerd/linkerd2", + "repository": "n8n-io/n8n", "pins": [ { "path": "upstream", - "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", + "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", "kind": "gitlink" } ] }, - "plan_sha256": "ae86ce031289490b5b7c93f169fa0308d90cec5cd3e0281f4dce333e386f427a", - "definition_sha256": "e55db4e3f59df4ea63c2886c2e8189f56ee238c9fb14f0eea7bbabaeaa3c83ef", + "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9", + "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, + "valid_for_comparison": true, "execution_verified": true, - "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [], "summaries": [ { @@ -4978,9 +7682,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 306, - "min": 306, - "max": 306 + "median": 125, + "min": 125, + "max": 125 }, "storage": null, "storage_measured_count": 0 @@ -4990,26 +7694,30 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 15, - "min": 15, - "max": 15 + "median": 4, + "min": 4, + "max": 4 }, - "storage": null, - "storage_measured_count": 0 + "storage": { + "median": 52937490, + "min": 52937490, + "max": 52937490 + }, + "storage_measured_count": 1 }, { "phase": "cold", "provider": "boringcache", "count": 1, "measurement": { - "median": 200, - "min": 200, - "max": 200 + "median": 132, + "min": 132, + "max": 132 }, "storage": { - "median": 1230761384, - "min": 1230761384, - "max": 1230761384 + "median": 46409937, + "min": 46409937, + "max": 46409937 }, "storage_measured_count": 1 }, @@ -5018,77 +7726,78 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 12, - "min": 12, - "max": 12 + "median": 9, + "min": 9, + "max": 9 }, "storage": { - "median": 1230761384, - "min": 1230761384, - "max": 1230761384 + "median": 51597365, + "min": 51597365, + "max": 51597365 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "linkerd2", + "case_id": "n8n", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "linkerd2", + "case_id": "n8n", "cli_version": "v1.33.0", - "series_id": "image-output-qualification-01", + "variant": "turbo", + "series_id": "shared-turbo-qualification-01", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "native-workload-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-03T06:54:52Z", - "run_id": 37104596700, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700" + "requested_at": "2026-10-03T01:46:05Z", + "run_id": 37087350856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856" } ], "completions": [ { "schema_version": 1, - "case_id": "linkerd2", - "series_id": "image-output-qualification-01", + "case_id": "n8n", + "series_id": "shared-turbo-qualification-01", "sample": 1, - "run_id": "37104596700", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104596700", - "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "run_id": "37087350856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856", + "source_sha": "6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "3eba28538f46405b75d696f6c8071d56e26bf96e6b1be1f7fc5479893e282adf" + "evidence_manifest_sha256": "25fcd514b9aa761dbc44baa4a988e563bf43e071a2538617921e2c23a7e5c5b0" } ], "report_state": "current", - "report_path": "results/linkerd2/image-output-qualification-01/report.json", + "report_path": "results/n8n/shared-turbo-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104596700, + "run_id": 37087350856, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 15, "gaps": [], "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104596700.tar", - "bundle_bytes": 354816, - "sha256": "63efd14ac9e14ffd6b09b3ec57786f72aba962deda6f19fb773296b7428e1a60", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-n8n-turbo-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-n8n-turbo-2026-10-03/benchmarks-37087350856.tar", + "bundle_bytes": 484864, + "sha256": "8ebeab6ffdf273e07caa14b311620c1889a43ba722f2b4ce64e592237835e383", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "linkerd2", - "series_id": "migration-screening-01", - "plan_path": "results/linkerd2/migration-screening-01/series.json", + "case_id": "obs-studio", + "series_id": "direct-ccache-qualification", + "plan_path": "results/obs-studio/direct-ccache-qualification/series.json", "lane": "fresh", - "variant": null, + "variant": "ccache", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -5096,24 +7805,29 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "linkerd/linkerd2", + "repository": "obsproject/obs-studio", "pins": [ { - "path": "upstream", - "revision": "e7c15f3a6c933d4d92a4959575d5f9d093fdffc5", - "kind": "gitlink" + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" } ] }, - "plan_sha256": "8dc741e2abd94953818d2db8a9f0d4648b7ca290e1f1972e87be1fa0c9eb3cd5", - "definition_sha256": "9458671ff40345d67431c0b0cdb14a6d4e7c6ca38a2a40766e11eb85ee3eae02", + "plan_sha256": "43be5df991563240acd9e992a005ed4185648978c059cae677c7e0fd97c27d31", + "definition_sha256": "c6884e149cd5173da069bef0b27f987dd82be8f48814a51c0a2cb2a265beb763", "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, @@ -5132,77 +7846,124 @@ ], [ 1, - "warm", + "commit", "boringcache" ], [ 1, - "warm", + "commit", "actions-cache" ] ], "summaries": [], "dispatches": [ { - "case_id": "linkerd2", + "case_id": "obs-studio", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "obs-studio-obs-actions-cache.yml", "lane": "fresh", "inputs": { - "case_id": "linkerd2", "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "cache_tool": "ccache", + "series_id": "direct-ccache-qualification", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "benchmark-qualification-2026-10-02", "state": "requested", - "requested_at": "2026-10-03T01:14:40Z", - "run_id": 37085339156, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085339156" + "requested_at": "2026-10-02T20:53:34Z", + "run_id": 37063467317, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063467317" + }, + { + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-studio-obs-boringcache.yml", + "lane": "fresh", + "inputs": { + "cli_version": "v1.33.0", + "cache_tool": "ccache", + "series_id": "direct-ccache-qualification", + "sample": "1" + }, + "ref": "benchmark-qualification-2026-10-02", + "state": "requested", + "requested_at": "2026-10-02T20:53:32Z", + "run_id": 37063464018, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063464018" } ], "completions": [ { "schema_version": 1, - "case_id": "linkerd2", - "series_id": "migration-screening-01", + "case_id": "obs-studio", + "series_id": "direct-ccache-qualification", "sample": 1, - "run_id": "37085339156", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085339156", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", - "github_conclusion": "cancelled", + "run_id": "37063464018", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063464018", + "source_sha": "50f069f123cef134e307db653c853d39eee08404", + "github_conclusion": "failure", "status": "failed", "errors": [ - "Workflow concluded cancelled" + "Workflow concluded failure", + "BoringCache OBS ccache cold: failure" ], - "evidence_manifest_sha256": "91fd2da3d5203e0dac84d707beb7774c7689536058904f21baf6f27879260f68" + "evidence_manifest_sha256": "7cd3a299b99cf4772bc7377f60921eefa4cd62e00272b6d893b99c87409ad1fe" + }, + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "direct-ccache-qualification", + "sample": 1, + "run_id": "37063467317", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063467317", + "source_sha": "50f069f123cef134e307db653c853d39eee08404", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions OBS ccache cold: failure" + ], + "evidence_manifest_sha256": "78815b4b0e4a976eff37bf3f035c3952f003971727a4a6adcd072a532f42d064" } ], "report_state": "current", - "report_path": "results/linkerd2/migration-screening-01/report.json", + "report_path": "results/obs-studio/direct-ccache-qualification/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085339156, + "run_id": 37063464018, + "inventory_state": "complete", + "verified_files": 7, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, commit metadata, and workflow; no workload artifacts were produced", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-preparation-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-preparation-2026-10-02/benchmarks-37063464018.tar", + "bundle_bytes": 251392, + "sha256": "5aec5b72aedf003c0f71a98fd85abdec7a463a79b801d14073b5382223ceb82a", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37063467317, "inventory_state": "complete", "verified_files": 7, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085339156.tar", - "bundle_bytes": 87040, - "sha256": "f7d828874085433b11447c700c224bb24d3f7237a290a9cfdb31ef0ef4d52880", + "scope": "This run's available attempts, jobs, logs, commit metadata, and workflow; no workload artifacts were produced", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-preparation-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-preparation-2026-10-02/benchmarks-37063467317.tar", + "bundle_bytes": 247296, + "sha256": "d4b0215eef1e74e7707f77b7d732de6250c37df2c9c64e6637bffa6f399ffbb0", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "mastodon", - "series_id": "image-output-qualification-01", - "plan_path": "results/mastodon/image-output-qualification-01/series.json", + "case_id": "obs-studio", + "series_id": "direct-ccache-runtime-qualification", + "plan_path": "results/obs-studio/direct-ccache-runtime-qualification/series.json", "lane": "fresh", - "variant": "server", + "variant": "ccache", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -5210,32 +7971,34 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "mastodon/mastodon", + "repository": "obsproject/obs-studio", "pins": [ { - "path": "upstream", - "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", - "kind": "gitlink" + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" } ] }, - "plan_sha256": "c8ae9766cb832203e9984cda74e804fa1e746eecf842c469c2222f70f863d413", - "definition_sha256": "b3fdf15ac786c8f1a57ed78bfb7c3aa772d97f8624cb172cbf7443b80b8720ef", + "plan_sha256": "ffab8b2597d68c290ec801ec02818603718d78d6e3da18b74ec79f32646773bc", + "definition_sha256": "cb8f88cb8f67ec398827c90005ccd49a117d89ccf31e35977ed36a941f59b4e5", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, + "valid_for_comparison": true, "execution_verified": true, - "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [], "summaries": [ { @@ -5243,21 +8006,21 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 736, - "min": 736, - "max": 736 + "median": 486.020332187, + "min": 486.020332187, + "max": 486.020332187 }, "storage": null, "storage_measured_count": 0 }, { - "phase": "warm", + "phase": "commit", "provider": "actions-cache", "count": 1, "measurement": { - "median": 59, - "min": 59, - "max": 59 + "median": 37.84547679900001, + "min": 37.84547679900001, + "max": 37.84547679900001 }, "storage": null, "storage_measured_count": 0 @@ -5267,94 +8030,136 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 424, - "min": 424, - "max": 424 + "median": 489.142139402, + "min": 489.142139402, + "max": 489.142139402 }, "storage": { - "median": 1130445165, - "min": 1130445165, - "max": 1130445165 + "median": 172198996, + "min": 172198996, + "max": 172198996 }, "storage_measured_count": 1 }, { - "phase": "warm", + "phase": "commit", "provider": "boringcache", "count": 1, "measurement": { - "median": 38, - "min": 38, - "max": 38 + "median": 39.12283147599999, + "min": 39.12283147599999, + "max": 39.12283147599999 }, "storage": { - "median": 1130445165, - "min": 1130445165, - "max": 1130445165 + "median": 172198996, + "min": 172198996, + "max": 172198996 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "mastodon", + "case_id": "obs-studio", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "obs-studio-obs-actions-cache.yml", "lane": "fresh", "inputs": { - "case_id": "mastodon", "cli_version": "v1.33.0", - "variant": "server", - "series_id": "image-output-qualification-01", + "cache_tool": "ccache", + "series_id": "direct-ccache-runtime-qualification", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "benchmark-ruby-qualification-2026-10-02", "state": "requested", - "requested_at": "2026-10-03T06:54:55Z", - "run_id": 37104599574, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574" + "requested_at": "2026-10-02T21:17:50Z", + "run_id": 37065979394, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065979394" + }, + { + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-studio-obs-boringcache.yml", + "lane": "fresh", + "inputs": { + "cli_version": "v1.33.0", + "cache_tool": "ccache", + "series_id": "direct-ccache-runtime-qualification", + "sample": "1" + }, + "ref": "benchmark-ruby-qualification-2026-10-02", + "state": "requested", + "requested_at": "2026-10-02T21:17:13Z", + "run_id": 37065916863, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065916863" } ], "completions": [ { "schema_version": 1, - "case_id": "mastodon", - "series_id": "image-output-qualification-01", + "case_id": "obs-studio", + "series_id": "direct-ccache-runtime-qualification", "sample": 1, - "run_id": "37104599574", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104599574", - "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "run_id": "37065916863", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065916863", + "source_sha": "d69d3332963eb6728f33982ac8292f9f28f82fb6", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "2e2e3c52be0fbdb8c2ea7924416d0bc1530b9ee9b391a13b8f3fc03b0ab61e52" + "evidence_manifest_sha256": "5c0fcfcca196cefdc1a81e2462ee3d3a243bacf42197315ae5ad8d020e7c283b" + }, + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "direct-ccache-runtime-qualification", + "sample": 1, + "run_id": "37065979394", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065979394", + "source_sha": "d69d3332963eb6728f33982ac8292f9f28f82fb6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "357ed07d87e17cd0c70d427b1c906f040263e31ad984a41cf3386ce26559d6c8" } ], "report_state": "current", - "report_path": "results/mastodon/image-output-qualification-01/report.json", + "report_path": "results/obs-studio/direct-ccache-runtime-qualification/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104599574, + "run_id": 37065979394, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 9, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104599574.tar", - "bundle_bytes": 694784, - "sha256": "8b6a772ff39503d0e9aac23a96e13f627e9ffbba1f4e84ece45598857559ee60", + "scope": "Available attempts, jobs, logs, artifacts, commit metadata, and workflow for this run; not an original-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-ccache-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-ccache-2026-10-02/benchmarks-37065979394.tar", + "bundle_bytes": 591360, + "sha256": "68406fab4a85d2626e0caba1dd232049ff7176b2c4a34672e5272e28f04f693c", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37065916863, + "inventory_state": "complete", + "verified_files": 11, + "gaps": [], + "scope": "Available attempts, jobs, logs, artifacts, commit metadata, and workflow for this run; not an original-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-ccache-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-ccache-2026-10-02/benchmarks-37065916863.tar", + "bundle_bytes": 627200, + "sha256": "3e8273299f12ab4f5c2f9636ac63424a9922b1504a3d69bd1c636b725f8ee073", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "mastodon", - "series_id": "streaming-output-qualification-01", - "plan_path": "results/mastodon/streaming-output-qualification-01/series.json", + "case_id": "obs-studio", + "series_id": "direct-xcode-qualification", + "plan_path": "results/obs-studio/direct-xcode-qualification/series.json", "lane": "fresh", - "variant": "streaming", + "variant": "xcode", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -5362,32 +8167,34 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "mastodon/mastodon", + "repository": "obsproject/obs-studio", "pins": [ { - "path": "upstream", - "revision": "4995c7ccbe03ce84795fff3b9bdc899f675ab08c", - "kind": "gitlink" + "path": "payload/benchmark-source.env", + "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", + "kind": "env" + }, + { + "path": "payload/benchmark-source.env", + "revision": "ef5814431c6af9061874470201d0b799bc84602b", + "kind": "env" } ] }, - "plan_sha256": "3795707ead334fd02443574fd818023c3f29e42680b1cffe9693d54da38610af", - "definition_sha256": "6bd5fa5df953cc47d2b50d2e72997344cb1eb73532b5b0c7ecfc3e39c0d96ea5", + "plan_sha256": "e4844c0a8e0f0e62c33b4c96b883c4af866cd5cf1b96a2a322eaff8021389f4a", + "definition_sha256": "c6884e149cd5173da069bef0b27f987dd82be8f48814a51c0a2cb2a265beb763", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, + "valid_for_comparison": true, "execution_verified": true, - "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [], "summaries": [ { @@ -5395,21 +8202,21 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 52, - "min": 52, - "max": 52 + "median": 656.8639490000003, + "min": 656.8639490000003, + "max": 656.8639490000003 }, "storage": null, "storage_measured_count": 0 }, { - "phase": "warm", + "phase": "commit", "provider": "actions-cache", "count": 1, "measurement": { - "median": 21, - "min": 21, - "max": 21 + "median": 100.81440899999996, + "min": 100.81440899999996, + "max": 100.81440899999996 }, "storage": null, "storage_measured_count": 0 @@ -5419,302 +8226,429 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 40, - "min": 40, - "max": 40 + "median": 658.400308, + "min": 658.400308, + "max": 658.400308 }, "storage": { - "median": 106281953, - "min": 106281953, - "max": 106281953 + "median": 983997286, + "min": 983997286, + "max": 983997286 }, "storage_measured_count": 1 }, { - "phase": "warm", + "phase": "commit", "provider": "boringcache", "count": 1, "measurement": { - "median": 22, - "min": 22, - "max": 22 + "median": 193.70680499999997, + "min": 193.70680499999997, + "max": 193.70680499999997 }, "storage": { - "median": 106281953, - "min": 106281953, - "max": 106281953 + "median": 985937000, + "min": 985937000, + "max": 985937000 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "mastodon", + "case_id": "obs-studio", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "obs-studio-obs-actions-cache.yml", "lane": "fresh", "inputs": { - "case_id": "mastodon", "cli_version": "v1.33.0", - "variant": "streaming", - "series_id": "streaming-output-qualification-01", + "cache_tool": "xcode", + "series_id": "direct-xcode-qualification", "sample": "1" }, - "ref": "active-variant-qualification-2026-10-03", + "ref": "benchmark-qualification-2026-10-02", "state": "requested", - "requested_at": "2026-10-03T19:45:29Z", - "run_id": 37149053291, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291" + "requested_at": "2026-10-02T20:53:38Z", + "run_id": 37063474452, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063474452" + }, + { + "case_id": "obs-studio", + "repository": "boringcache/benchmarks", + "workflow": "obs-studio-obs-boringcache.yml", + "lane": "fresh", + "inputs": { + "cli_version": "v1.33.0", + "cache_tool": "xcode", + "series_id": "direct-xcode-qualification", + "sample": "1" + }, + "ref": "benchmark-qualification-2026-10-02", + "state": "requested", + "requested_at": "2026-10-02T20:53:36Z", + "run_id": 37063471110, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063471110" } ], "completions": [ { "schema_version": 1, - "case_id": "mastodon", - "series_id": "streaming-output-qualification-01", + "case_id": "obs-studio", + "series_id": "direct-xcode-qualification", "sample": 1, - "run_id": "37149053291", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149053291", - "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "run_id": "37063471110", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063471110", + "source_sha": "50f069f123cef134e307db653c853d39eee08404", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "530bf111ae6b4a6f70347e2e293b0ad75cec21dd175728be464cf8ecfc1ebfec" + "evidence_manifest_sha256": "0796f950e9144c52105fa2b0850aeac7b5b703d8b8adaae2fcab8795af410897" + }, + { + "schema_version": 1, + "case_id": "obs-studio", + "series_id": "direct-xcode-qualification", + "sample": 1, + "run_id": "37063474452", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063474452", + "source_sha": "50f069f123cef134e307db653c853d39eee08404", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "0b9e1dcef017afdb0ad148f037682a33c3cde8edaab05d1f22502190325df3eb" } ], "report_state": "current", - "report_path": "results/mastodon/streaming-output-qualification-01/report.json", + "report_path": "results/obs-studio/direct-xcode-qualification/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37149053291, + "run_id": 37063474452, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 9, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149053291.tar", - "bundle_bytes": 266240, - "sha256": "899c7adc2f77d143cfdda44d59b9f12884877c5a5fcf65bed1c5989341c8b2a9", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of the original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-xcode-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-xcode-2026-10-02/benchmarks-37063474452.tar", + "bundle_bytes": 551424, + "sha256": "96a08a7bd4e57db69820bd77169440b1e9f2cd3845835a53c0f27e6cb068f9a2", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37063471110, + "inventory_state": "complete", + "verified_files": 11, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of the original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-xcode-2026-10-02", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-xcode-2026-10-02/benchmarks-37063471110.tar", + "bundle_bytes": 590848, + "sha256": "d1b342fadf7c9d3bb561fa84829b11cde4a7b90a5710d5ccc6f81142d0c4b492", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "n8n", - "series_id": "distroless-output-qualification-01", - "plan_path": "results/n8n/distroless-output-qualification-01/series.json", + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "plan_path": "results/opencut-moon/registry-qualification-01/series.json", "lane": "fresh", - "variant": "distroless", - "sample_count": 1, + "variant": null, + "sample_count": 2, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Moon restore the OpenCut web build on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "n8n-io/n8n", + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", "pins": [ { - "path": "upstream", - "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" } ] }, - "plan_sha256": "08b518d5c7cd45195e1817738316c065e60de1805ae26927f3b1c572b2aeef55", - "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "plan_sha256": "726ca855e2aad904fe6a34e52fb84e252e35aa6504556537fa0f7c979f9c0bc7", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": true, "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + "This series does not declare a comparison of provider performance." ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 214, - "min": 214, - "max": 214 - }, + "provider": "bazel-remote", + "count": 2, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 132, - "min": 132, - "max": 132 - }, + "provider": "bazel-remote", + "count": 2, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", - "count": 1, - "measurement": { - "median": 81, - "min": 81, - "max": 81 - }, - "storage": { - "median": 581055738, - "min": 581055738, - "max": 581055738 - }, - "storage_measured_count": 1 + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", "provider": "boringcache", - "count": 1, - "measurement": { - "median": 110, - "min": 110, - "max": 110 - }, - "storage": { - "median": 581055738, - "min": 581055738, - "max": 581055738 - }, - "storage_measured_count": 1 + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "n8n", + "case_id": "opencut-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "n8n", - "cli_version": "v1.33.0", - "variant": "distroless", - "series_id": "distroless-output-qualification-01", + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", "sample": "1" }, - "ref": "active-variant-qualification-2026-10-03", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-03T19:45:33Z", - "run_id": 37149056843, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843" + "requested_at": "2026-10-05T11:26:19Z", + "run_id": 37302929873, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + }, + { + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:26:21Z", + "run_id": 37302932865, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302932865" } ], "completions": [ { "schema_version": 1, - "case_id": "n8n", - "series_id": "distroless-output-qualification-01", + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", "sample": 1, - "run_id": "37149056843", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149056843", - "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "run_id": "37302929873", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302929873", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "4bbc22e12b0a3e5e9c1c5c481f68f29002291e60a1bcc748ea3e18eaaed8566e" + "evidence_manifest_sha256": "c0e9cb3efb9b12c5aa2c25f24461e255ca60f47a2580ff454898e4e31b866b23" + }, + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302932865", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302932865", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "55fb7c3f486e1b081674b0bbcbadea8161bcd8329dc144c2946e65686b07f82f" } ], "report_state": "current", - "report_path": "results/n8n/distroless-output-qualification-01/report.json", + "report_path": "results/opencut-moon/registry-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37149056843, + "run_id": 37302929873, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149056843.tar", - "bundle_bytes": 1658880, - "sha256": "b1b97061460c375759d37184868d53cddad9c1b980b4e69d8e6a67cce374ce09", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302929873.tar.gz", + "bundle_bytes": 824311, + "sha256": "3231e868af54cc889150817a79d4899af7f3ea064ba4407d061a92fd139ff945", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302932865, + "inventory_state": "complete", + "verified_files": 16, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302932865.tar.gz", + "bundle_bytes": 824072, + "sha256": "c86bc6a965966dc545dffb405efff441a57143d0ef795e75f05543cdbdcf3093", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "n8n", - "series_id": "image-output-qualification-01", - "plan_path": "results/n8n/image-output-qualification-01/series.json", + "case_id": "opencut-moon", + "series_id": "registry-screening-01", + "plan_path": "results/opencut-moon/registry-screening-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "plan_sha256": "dcef2403e83603a0c55dcaba78d5eb8374c2081b82b8ade78853270575b59799", + "definition_sha256": "499d217459705cbbb0b18b1a70570c04ebc72db19dfd6f2cc37e292abd141a0f", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "opencut-moon", + "series_id": "registry-screening-02", + "plan_path": "results/opencut-moon/registry-screening-02/series.json", "lane": "fresh", - "variant": "docker", + "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Moon restore the OpenCut web build on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "n8n-io/n8n", + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", "pins": [ { - "path": "upstream", - "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" } ] }, - "plan_sha256": "b37a573111c573f83938292e8aba01df839799e18ea821f07dadca5e869b74a5", - "definition_sha256": "009c1fb0a0a062a44f7a4aee84a48e3570585410de8bce2bc23fee0de426aa24", - "state": "completed", + "plan_sha256": "1cb918b67aa5053d594ead632e738134735a9a7d9b949a4d4291b2768ef0f1f8", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": true, + "execution_verified": false, "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "warm", + "boringcache" + ] ], - "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 446, - "min": 446, - "max": 446 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 150, - "min": 150, - "max": 150 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, @@ -5722,151 +8656,190 @@ "phase": "cold", "provider": "boringcache", "count": 1, - "measurement": { - "median": 319, - "min": 319, - "max": 319 - }, - "storage": { - "median": 483433384, - "min": 483433384, - "max": 483433384 - }, - "storage_measured_count": 1 - }, - { - "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 162, - "min": 162, - "max": 162 - }, - "storage": { - "median": 483433384, - "min": 483433384, - "max": 483433384 - }, - "storage_measured_count": 1 + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "n8n", + "case_id": "opencut-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "n8n", - "cli_version": "v1.33.0", - "variant": "docker", - "series_id": "image-output-qualification-01", + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "main", "state": "requested", - "requested_at": "2026-10-03T06:54:54Z", - "run_id": 37104598178, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178" + "requested_at": "2026-10-05T11:17:34Z", + "run_id": 37302001164, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164" } ], "completions": [ { "schema_version": 1, - "case_id": "n8n", - "series_id": "image-output-qualification-01", + "case_id": "opencut-moon", + "series_id": "registry-screening-02", "sample": 1, - "run_id": "37104598178", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104598178", - "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "5f0af1311b0f43733e4562504aed6f42dde1420f1c800abe59315d5f97a55214" + "run_id": "37302001164", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache opencut-moon warm: failure" + ], + "evidence_manifest_sha256": "09e17dc66880bf03926e144770b393e02d183ee23c1675b0d1a3854ebdb6d5d6" } ], "report_state": "current", - "report_path": "results/n8n/image-output-qualification-01/report.json", + "report_path": "results/opencut-moon/registry-screening-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104598178, + "run_id": 37302001164, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 15, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104598178.tar", - "bundle_bytes": 1748480, - "sha256": "0bb9fc327d8fef9a8af1eb8bdcf8ec699508a46715bae610a28c12b0fc1dd49f", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302001164.tar.gz", + "bundle_bytes": 841387, + "sha256": "64aeb093857a8b1dcab08060ddf79c4ce8b85af3f39579141071ea91cfc42e7c", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "n8n", - "series_id": "runners-output-qualification-01", - "plan_path": "results/n8n/runners-output-qualification-01/series.json", + "case_id": "opencut-moon", + "series_id": "registry-screening-03", + "plan_path": "results/opencut-moon/registry-screening-03/series.json", "lane": "fresh", - "variant": "runners", + "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Moon restore the OpenCut web build on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Selected workload build plus measured provider setup: Turbo excludes dependency installation; Docker includes image export and load and excludes prepared source setup. Output checks and unmeasured post-job work are reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "n8n-io/n8n", + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", "pins": [ { - "path": "upstream", - "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" } ] }, - "plan_sha256": "9ce0493c6b520b2ecadc9a13123edee5590770d648062b0376a8195ee06f489e", - "definition_sha256": "454acd4311833555c8086d1e9437e3cb618f6dfd3eadf93c9757aa51816c68b2", + "plan_sha256": "228b7e3a051d81a1cb57a81e265507e7e6beef5c8ee0181fd54dc9e1f71b43ae", + "definition_sha256": "d804908655d0ff3aae18750d5c32c8c4c74b07fc140fbd8d5a2cc8ac3feb6d87", + "state": "planned", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], + "runs_without_archives": [] + }, + { + "case_id": "opencut-moon", + "series_id": "registry-screening-04", + "plan_path": "results/opencut-moon/registry-screening-04/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "plan_sha256": "7016af095356764099bf0da79cec88f91affb18267ec18b5ebebdc310acdff72", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": true, "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + "This series does not declare a comparison of provider performance." ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 144, - "min": 144, - "max": 144 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 89, - "min": 89, - "max": 89 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, @@ -5874,95 +8847,78 @@ "phase": "cold", "provider": "boringcache", "count": 1, - "measurement": { - "median": 62, - "min": 62, - "max": 62 - }, - "storage": { - "median": 413827357, - "min": 413827357, - "max": 413827357 - }, - "storage_measured_count": 1 + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 60, - "min": 60, - "max": 60 - }, - "storage": { - "median": 413827357, - "min": 413827357, - "max": 413827357 - }, - "storage_measured_count": 1 + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "n8n", + "case_id": "opencut-moon", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "n8n", - "cli_version": "v1.33.0", - "variant": "runners", - "series_id": "runners-output-qualification-01", + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", "sample": "1" }, - "ref": "active-variant-qualification-2026-10-03", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-03T19:45:31Z", - "run_id": 37149055130, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130" + "requested_at": "2026-10-05T11:22:04Z", + "run_id": 37302476504, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302476504" } ], "completions": [ { "schema_version": 1, - "case_id": "n8n", - "series_id": "runners-output-qualification-01", + "case_id": "opencut-moon", + "series_id": "registry-screening-04", "sample": 1, - "run_id": "37149055130", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37149055130", - "source_sha": "b759879316279afa06a7e8166c905429547adb34", + "run_id": "37302476504", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302476504", + "source_sha": "a15241e5d23058d6fa8890ab4e16fd15e319d417", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "c52e6d5bedd2e1d5e4333c820e0895b30eee8eb5c5ec02389cb5d669dc6f7017" + "evidence_manifest_sha256": "958d86ab1a602a20ac8b3b656219b3b598ff8672fea1e5a947f6e520f77c796c" } ], "report_state": "current", - "report_path": "results/n8n/runners-output-qualification-01/report.json", + "report_path": "results/opencut-moon/registry-screening-04/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37149055130, + "run_id": 37302476504, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37149055130.tar", - "bundle_bytes": 1602560, - "sha256": "36d2b8fe86fbfe42fa9df155468b0beda3b6a1baabec398dd3a577e04feafafd", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302476504.tar.gz", + "bundle_bytes": 822268, + "sha256": "6860e07a5f7ead5a10198483d4c4f9535ee54f54f4041bb6681a87a701b136f6", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "n8n", - "series_id": "shared-turbo-qualification-01", - "plan_path": "results/n8n/shared-turbo-qualification-01/series.json", + "case_id": "opentelemetry-java", + "series_id": "migration-screening-01", + "plan_path": "results/opentelemetry-java/migration-screening-01/series.json", "lane": "fresh", - "variant": "turbo", + "variant": null, "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -5977,17 +8933,17 @@ "sample_count": 10 }, "source": { - "repository": "n8n-io/n8n", + "repository": "open-telemetry/opentelemetry-java", "pins": [ { "path": "upstream", - "revision": "7576de5f26ccfd20c1aceee0e6f5657833e267d9", + "revision": "a674ff1acea16754a2ed413fb8966084220fbd8b", "kind": "gitlink" } ] }, - "plan_sha256": "035df772551717a7a2d369d39994c744f270815a5481833e7a494a06097dcae9", - "definition_sha256": "c2930c6848f23cb393381f93848cea9037caba566e8e476405c4b40cb2329ec3", + "plan_sha256": "7d185a491dea81a5681ed1bf9f7b14e76c883193fee3d4441c63c37c533d1ecc", + "definition_sha256": "e1911882d495ee833991933d9b24cc819914f7b41072532fa8fe97e6d685c22a", "state": "completed", "publication": "unreviewed", "valid_for_comparison": true, @@ -6000,9 +8956,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 125, - "min": 125, - "max": 125 + "median": 966, + "min": 966, + "max": 966 }, "storage": null, "storage_measured_count": 0 @@ -6012,14 +8968,14 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 4, - "min": 4, - "max": 4 + "median": 171, + "min": 171, + "max": 171 }, "storage": { - "median": 52937490, - "min": 52937490, - "max": 52937490 + "median": 50739407, + "min": 50739407, + "max": 50739407 }, "storage_measured_count": 1 }, @@ -6028,14 +8984,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 132, - "min": 132, - "max": 132 + "median": 975, + "min": 975, + "max": 975 }, "storage": { - "median": 46409937, - "min": 46409937, - "max": 46409937 + "median": 51294699, + "min": 51294699, + "max": 51294699 }, "storage_measured_count": 1 }, @@ -6044,244 +9000,77 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 9, - "min": 9, - "max": 9 + "median": 150, + "min": 150, + "max": 150 }, "storage": { - "median": 51597365, - "min": 51597365, - "max": 51597365 + "median": 51294699, + "min": 51294699, + "max": 51294699 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "n8n", + "case_id": "opentelemetry-java", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "n8n", - "cli_version": "v1.33.0", - "variant": "turbo", - "series_id": "shared-turbo-qualification-01", - "sample": "1" - }, - "ref": "native-workload-qualification-2026-10-03", - "state": "requested", - "requested_at": "2026-10-03T01:46:05Z", - "run_id": 37087350856, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856" - } - ], - "completions": [ - { - "schema_version": 1, - "case_id": "n8n", - "series_id": "shared-turbo-qualification-01", - "sample": 1, - "run_id": "37087350856", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37087350856", - "source_sha": "6eda03c19ffe66ff3d0d6b3d9f76fb2a2860ac0c", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "25fcd514b9aa761dbc44baa4a988e563bf43e071a2538617921e2c23a7e5c5b0" - } - ], - "report_state": "current", - "report_path": "results/n8n/shared-turbo-qualification-01/report.json", - "evidence": [ - { - "repository": "boringcache/benchmarks", - "run_id": 37087350856, - "inventory_state": "complete", - "verified_files": 15, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-n8n-turbo-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-n8n-turbo-2026-10-03/benchmarks-37087350856.tar", - "bundle_bytes": 484864, - "sha256": "8ebeab6ffdf273e07caa14b311620c1889a43ba722f2b4ce64e592237835e383", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - } - ], - "runs_without_archives": [] - }, - { - "case_id": "obs-studio", - "series_id": "direct-ccache-qualification", - "plan_path": "results/obs-studio/direct-ccache-qualification/series.json", - "lane": "fresh", - "variant": "ccache", - "sample_count": 1, - "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", - "providers": [ - "boringcache", - "actions-cache" - ], - "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 - }, - "source": { - "repository": "obsproject/obs-studio", - "pins": [ - { - "path": "payload/benchmark-source.env", - "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", - "kind": "env" - }, - { - "path": "payload/benchmark-source.env", - "revision": "ef5814431c6af9061874470201d0b799bc84602b", - "kind": "env" - } - ] - }, - "plan_sha256": "43be5df991563240acd9e992a005ed4185648978c059cae677c7e0fd97c27d31", - "definition_sha256": "c6884e149cd5173da069bef0b27f987dd82be8f48814a51c0a2cb2a265beb763", - "state": "failed", - "publication": "unreviewed", - "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "commit", - "boringcache" - ], - [ - 1, - "commit", - "actions-cache" - ] - ], - "summaries": [], - "dispatches": [ - { - "case_id": "obs-studio", - "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-actions-cache.yml", - "lane": "fresh", - "inputs": { - "cli_version": "v1.33.0", - "cache_tool": "ccache", - "series_id": "direct-ccache-qualification", - "sample": "1" - }, - "ref": "benchmark-qualification-2026-10-02", - "state": "requested", - "requested_at": "2026-10-02T20:53:34Z", - "run_id": 37063467317, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063467317" - }, - { - "case_id": "obs-studio", - "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-boringcache.yml", - "lane": "fresh", - "inputs": { + "case_id": "opentelemetry-java", "cli_version": "v1.33.0", - "cache_tool": "ccache", - "series_id": "direct-ccache-qualification", + "series_id": "migration-screening-01", "sample": "1" }, - "ref": "benchmark-qualification-2026-10-02", + "ref": "active-case-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-02T20:53:32Z", - "run_id": 37063464018, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063464018" + "requested_at": "2026-10-03T01:14:41Z", + "run_id": 37085341225, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225" } ], "completions": [ { "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-ccache-qualification", - "sample": 1, - "run_id": "37063464018", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063464018", - "source_sha": "50f069f123cef134e307db653c853d39eee08404", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "BoringCache OBS ccache cold: failure" - ], - "evidence_manifest_sha256": "7cd3a299b99cf4772bc7377f60921eefa4cd62e00272b6d893b99c87409ad1fe" - }, - { - "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-ccache-qualification", + "case_id": "opentelemetry-java", + "series_id": "migration-screening-01", "sample": 1, - "run_id": "37063467317", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063467317", - "source_sha": "50f069f123cef134e307db653c853d39eee08404", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "GitHub Actions OBS ccache cold: failure" - ], - "evidence_manifest_sha256": "78815b4b0e4a976eff37bf3f035c3952f003971727a4a6adcd072a532f42d064" + "run_id": "37085341225", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "e6bd8885e790a8be05bb2a0ed57a46aeb8ba5c8b40829917e5340a1e7d3a9f38" } ], "report_state": "current", - "report_path": "results/obs-studio/direct-ccache-qualification/report.json", + "report_path": "results/opentelemetry-java/migration-screening-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37063464018, - "inventory_state": "complete", - "verified_files": 7, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, commit metadata, and workflow; no workload artifacts were produced", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-preparation-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-preparation-2026-10-02/benchmarks-37063464018.tar", - "bundle_bytes": 251392, - "sha256": "5aec5b72aedf003c0f71a98fd85abdec7a463a79b801d14073b5382223ceb82a", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37063467317, + "run_id": 37085341225, "inventory_state": "complete", - "verified_files": 7, + "verified_files": 15, "gaps": [], - "scope": "This run's available attempts, jobs, logs, commit metadata, and workflow; no workload artifacts were produced", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-preparation-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-preparation-2026-10-02/benchmarks-37063467317.tar", - "bundle_bytes": 247296, - "sha256": "d4b0215eef1e74e7707f77b7d732de6250c37df2c9c64e6637bffa6f399ffbb0", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085341225.tar", + "bundle_bytes": 472576, + "sha256": "c5de94bda90ebb24e156895209fc648c5ad7d05cedbd7fb9cae1cd5ec67bff5c", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "obs-studio", - "series_id": "direct-ccache-runtime-qualification", - "plan_path": "results/obs-studio/direct-ccache-runtime-qualification/series.json", + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", + "plan_path": "results/posthog/image-output-combined-qualification-01/series.json", "lane": "fresh", - "variant": "ccache", + "variant": "combined", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -6289,34 +9078,32 @@ "boringcache", "actions-cache" ], - "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "obsproject/obs-studio", + "repository": "PostHog/posthog", "pins": [ { - "path": "payload/benchmark-source.env", - "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", - "kind": "env" - }, - { - "path": "payload/benchmark-source.env", - "revision": "ef5814431c6af9061874470201d0b799bc84602b", - "kind": "env" + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" } ] }, - "plan_sha256": "ffab8b2597d68c290ec801ec02818603718d78d6e3da18b74ec79f32646773bc", - "definition_sha256": "cb8f88cb8f67ec398827c90005ccd49a117d89ccf31e35977ed36a941f59b4e5", + "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -6324,21 +9111,21 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 486.020332187, - "min": 486.020332187, - "max": 486.020332187 + "median": 1889, + "min": 1889, + "max": 1889 }, "storage": null, "storage_measured_count": 0 }, { - "phase": "commit", + "phase": "warm", "provider": "actions-cache", "count": 1, "measurement": { - "median": 37.84547679900001, - "min": 37.84547679900001, - "max": 37.84547679900001 + "median": 187, + "min": 187, + "max": 187 }, "storage": null, "storage_measured_count": 0 @@ -6348,136 +9135,94 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 489.142139402, - "min": 489.142139402, - "max": 489.142139402 + "median": 1016, + "min": 1016, + "max": 1016 }, "storage": { - "median": 172198996, - "min": 172198996, - "max": 172198996 + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 }, "storage_measured_count": 1 }, { - "phase": "commit", + "phase": "warm", "provider": "boringcache", "count": 1, "measurement": { - "median": 39.12283147599999, - "min": 39.12283147599999, - "max": 39.12283147599999 + "median": 162, + "min": 162, + "max": 162 }, "storage": { - "median": 172198996, - "min": 172198996, - "max": 172198996 + "median": 6237909312, + "min": 6237909312, + "max": 6237909312 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "obs-studio", - "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-actions-cache.yml", - "lane": "fresh", - "inputs": { - "cli_version": "v1.33.0", - "cache_tool": "ccache", - "series_id": "direct-ccache-runtime-qualification", - "sample": "1" - }, - "ref": "benchmark-ruby-qualification-2026-10-02", - "state": "requested", - "requested_at": "2026-10-02T21:17:50Z", - "run_id": 37065979394, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065979394" - }, - { - "case_id": "obs-studio", + "case_id": "posthog", "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-boringcache.yml", + "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { + "case_id": "posthog", "cli_version": "v1.33.0", - "cache_tool": "ccache", - "series_id": "direct-ccache-runtime-qualification", + "variant": "combined", + "series_id": "image-output-combined-qualification-01", "sample": "1" }, - "ref": "benchmark-ruby-qualification-2026-10-02", + "ref": "docker-output-correction-2026-10-03", "state": "requested", - "requested_at": "2026-10-02T21:17:13Z", - "run_id": 37065916863, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065916863" + "requested_at": "2026-10-03T19:34:21Z", + "run_id": 37148377350, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" } ], "completions": [ { "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-ccache-runtime-qualification", - "sample": 1, - "run_id": "37065916863", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065916863", - "source_sha": "d69d3332963eb6728f33982ac8292f9f28f82fb6", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "5c0fcfcca196cefdc1a81e2462ee3d3a243bacf42197315ae5ad8d020e7c283b" - }, - { - "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-ccache-runtime-qualification", + "case_id": "posthog", + "series_id": "image-output-combined-qualification-01", "sample": 1, - "run_id": "37065979394", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37065979394", - "source_sha": "d69d3332963eb6728f33982ac8292f9f28f82fb6", + "run_id": "37148377350", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "357ed07d87e17cd0c70d427b1c906f040263e31ad984a41cf3386ce26559d6c8" + "evidence_manifest_sha256": "7d2cf7f6e9b12c476c9b3e23b73d31fbe8f25176a91bf7d18fc29a027e819895" } ], "report_state": "current", - "report_path": "results/obs-studio/direct-ccache-runtime-qualification/report.json", + "report_path": "results/posthog/image-output-combined-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37065979394, - "inventory_state": "complete", - "verified_files": 9, - "gaps": [], - "scope": "Available attempts, jobs, logs, artifacts, commit metadata, and workflow for this run; not an original-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-ccache-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-ccache-2026-10-02/benchmarks-37065979394.tar", - "bundle_bytes": 591360, - "sha256": "68406fab4a85d2626e0caba1dd232049ff7176b2c4a34672e5272e28f04f693c", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37065916863, + "run_id": 37148377350, "inventory_state": "complete", - "verified_files": 11, + "verified_files": 13, "gaps": [], - "scope": "Available attempts, jobs, logs, artifacts, commit metadata, and workflow for this run; not an original-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-ccache-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-ccache-2026-10-02/benchmarks-37065916863.tar", - "bundle_bytes": 627200, - "sha256": "3e8273299f12ab4f5c2f9636ac63424a9922b1504a3d69bd1c636b725f8ee073", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148377350.tar", + "bundle_bytes": 373248, + "sha256": "bf89e6b23b5517ca298ff620dde1885fce19ec9a9a8d92fce5092a2dfd93e893", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "obs-studio", - "series_id": "direct-xcode-qualification", - "plan_path": "results/obs-studio/direct-xcode-qualification/series.json", + "case_id": "posthog", + "series_id": "image-output-qualification-01", + "plan_path": "results/posthog/image-output-qualification-01/series.json", "lane": "fresh", - "variant": "xcode", + "variant": "layers", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -6485,56 +9230,55 @@ "boringcache", "actions-cache" ], - "timed_scope": "Cache restore/setup plus the upstream Building obs-studio log group; dependency installation, CMake configuration, queueing, and post-job cache save are excluded", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "obsproject/obs-studio", + "repository": "PostHog/posthog", "pins": [ { - "path": "payload/benchmark-source.env", - "revision": "e7136951c052db9a74de853f2f0ec5d1bc8f6f4a", - "kind": "env" - }, - { - "path": "payload/benchmark-source.env", - "revision": "ef5814431c6af9061874470201d0b799bc84602b", - "kind": "env" + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" } ] }, - "plan_sha256": "e4844c0a8e0f0e62c33b4c96b883c4af866cd5cf1b96a2a322eaff8021389f4a", - "definition_sha256": "c6884e149cd5173da069bef0b27f987dd82be8f48814a51c0a2cb2a265beb763", - "state": "completed", + "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", + "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "state": "failed", "publication": "unreviewed", - "valid_for_comparison": true, - "execution_verified": true, - "methodology_issues": [], - "missing_observations": [], + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", + "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series.", + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [ + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], "summaries": [ { "phase": "cold", "provider": "actions-cache", "count": 1, "measurement": { - "median": 656.8639490000003, - "min": 656.8639490000003, - "max": 656.8639490000003 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "commit", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 100.81440899999996, - "min": 100.81440899999996, - "max": 100.81440899999996 + "median": 1579, + "min": 1579, + "max": 1579 }, "storage": null, "storage_measured_count": 0 @@ -6544,136 +9288,82 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 658.400308, - "min": 658.400308, - "max": 658.400308 - }, - "storage": { - "median": 983997286, - "min": 983997286, - "max": 983997286 - }, - "storage_measured_count": 1 - }, - { - "phase": "commit", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 193.70680499999997, - "min": 193.70680499999997, - "max": 193.70680499999997 + "median": 813, + "min": 813, + "max": 813 }, "storage": { - "median": 985937000, - "min": 985937000, - "max": 985937000 + "median": 6237669195, + "min": 6237669195, + "max": 6237669195 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "obs-studio", - "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-actions-cache.yml", - "lane": "fresh", - "inputs": { - "cli_version": "v1.33.0", - "cache_tool": "xcode", - "series_id": "direct-xcode-qualification", - "sample": "1" - }, - "ref": "benchmark-qualification-2026-10-02", - "state": "requested", - "requested_at": "2026-10-02T20:53:38Z", - "run_id": 37063474452, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063474452" - }, - { - "case_id": "obs-studio", + "case_id": "posthog", "repository": "boringcache/benchmarks", - "workflow": "obs-studio-obs-boringcache.yml", + "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { + "case_id": "posthog", "cli_version": "v1.33.0", - "cache_tool": "xcode", - "series_id": "direct-xcode-qualification", + "variant": "layers", + "series_id": "image-output-qualification-01", "sample": "1" }, - "ref": "benchmark-qualification-2026-10-02", + "ref": "image-output-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-02T20:53:36Z", - "run_id": 37063471110, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063471110" + "requested_at": "2026-10-03T06:54:57Z", + "run_id": 37104600856, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" } ], "completions": [ { "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-xcode-qualification", - "sample": 1, - "run_id": "37063471110", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063471110", - "source_sha": "50f069f123cef134e307db653c853d39eee08404", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "0796f950e9144c52105fa2b0850aeac7b5b703d8b8adaae2fcab8795af410897" - }, - { - "schema_version": 1, - "case_id": "obs-studio", - "series_id": "direct-xcode-qualification", + "case_id": "posthog", + "series_id": "image-output-qualification-01", "sample": 1, - "run_id": "37063474452", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37063474452", - "source_sha": "50f069f123cef134e307db653c853d39eee08404", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "0b9e1dcef017afdb0ad148f037682a33c3cde8edaab05d1f22502190325df3eb" + "run_id": "37104600856", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", + "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "GitHub Actions posthog warm: failure", + "BoringCache posthog warm: failure" + ], + "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" } ], "report_state": "current", - "report_path": "results/obs-studio/direct-xcode-qualification/report.json", + "report_path": "results/posthog/image-output-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37063474452, + "run_id": 37104600856, "inventory_state": "complete", "verified_files": 9, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of the original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-xcode-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-xcode-2026-10-02/benchmarks-37063474452.tar", - "bundle_bytes": 551424, - "sha256": "96a08a7bd4e57db69820bd77169440b1e9f2cd3845835a53c0f27e6cb068f9a2", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - }, - { - "repository": "boringcache/benchmarks", - "run_id": 37063471110, - "inventory_state": "complete", - "verified_files": 11, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of the original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-obs-xcode-2026-10-02", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-obs-xcode-2026-10-02/benchmarks-37063471110.tar", - "bundle_bytes": 590848, - "sha256": "d1b342fadf7c9d3bb561fa84829b11cde4a7b90a5710d5ccc6f81142d0c4b492", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104600856.tar", + "bundle_bytes": 392704, + "sha256": "6a38f37864533c9921020e3d994a21914adf337775f42b734e74fd894eebc7e2", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "opentelemetry-java", - "series_id": "migration-screening-01", - "plan_path": "results/opentelemetry-java/migration-screening-01/series.json", + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "plan_path": "results/posthog/image-output-qualification-02/series.json", "lane": "fresh", - "variant": null, + "variant": "layers", "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -6681,29 +9371,32 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "open-telemetry/opentelemetry-java", + "repository": "PostHog/posthog", "pins": [ { "path": "upstream", - "revision": "a674ff1acea16754a2ed413fb8966084220fbd8b", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", "kind": "gitlink" } ] }, - "plan_sha256": "7d185a491dea81a5681ed1bf9f7b14e76c883193fee3d4441c63c37c533d1ecc", - "definition_sha256": "e1911882d495ee833991933d9b24cc819914f7b41072532fa8fe97e6d685c22a", + "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", + "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], "missing_observations": [], "summaries": [ { @@ -6711,9 +9404,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 966, - "min": 966, - "max": 966 + "median": 1609, + "min": 1609, + "max": 1609 }, "storage": null, "storage_measured_count": 0 @@ -6723,98 +9416,212 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 171, - "min": 171, - "max": 171 + "median": 175, + "min": 175, + "max": 175 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 778, + "min": 778, + "max": 778 }, "storage": { - "median": 50739407, - "min": 50739407, - "max": 50739407 + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 }, "storage_measured_count": 1 }, { - "phase": "cold", + "phase": "warm", "provider": "boringcache", "count": 1, "measurement": { - "median": 975, - "min": 975, - "max": 975 + "median": 136, + "min": 136, + "max": 136 }, "storage": { - "median": 51294699, - "min": 51294699, - "max": 51294699 + "median": 6237262375, + "min": 6237262375, + "max": 6237262375 }, "storage_measured_count": 1 - }, + } + ], + "dispatches": [ + { + "case_id": "posthog", + "repository": "boringcache/benchmarks", + "workflow": "native-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "posthog", + "cli_version": "v1.33.0", + "variant": "layers", + "series_id": "image-output-qualification-02", + "sample": "1" + }, + "ref": "docker-output-correction-2026-10-03", + "state": "requested", + "requested_at": "2026-10-03T19:34:19Z", + "run_id": 37148375181, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "posthog", + "series_id": "image-output-qualification-02", + "sample": 1, + "run_id": "37148375181", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181", + "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "ad47456c459688cb932e9d232c57790b64b694616f587246c3bb3b3c07e77e25" + } + ], + "report_state": "current", + "report_path": "results/posthog/image-output-qualification-02/report.json", + "evidence": [ { - "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 150, - "min": 150, - "max": 150 - }, - "storage": { - "median": 51294699, - "min": 51294699, - "max": 51294699 - }, - "storage_measured_count": 1 + "repository": "boringcache/benchmarks", + "run_id": 37148375181, + "inventory_state": "complete", + "verified_files": 13, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148375181.tar", + "bundle_bytes": 372736, + "sha256": "49df16499aee7a038dfea048f8ec82e605cbfa22fa8822e0060bc7ce213e65b4", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], + "runs_without_archives": [] + }, + { + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", + "plan_path": "results/posthog/shared-combined-lifecycle-01/series.json", + "lane": "fresh", + "variant": "combined", + "sample_count": 1, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache" + ], + "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "PostHog/posthog", + "pins": [ + { + "path": "upstream", + "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "kind": "gitlink" + } + ] + }, + "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7", + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", + "state": "failed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "actions-cache" + ] + ], + "summaries": [], "dispatches": [ { - "case_id": "opentelemetry-java", + "case_id": "posthog", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "opentelemetry-java", + "case_id": "posthog", "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "variant": "combined", + "series_id": "shared-combined-lifecycle-01", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "shared-lifecycle-2026-10-04", "state": "requested", - "requested_at": "2026-10-03T01:14:41Z", - "run_id": 37085341225, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225" + "requested_at": "2026-10-04T13:09:00Z", + "run_id": 37204588491, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491" } ], "completions": [ { "schema_version": 1, - "case_id": "opentelemetry-java", - "series_id": "migration-screening-01", + "case_id": "posthog", + "series_id": "shared-combined-lifecycle-01", "sample": 1, - "run_id": "37085341225", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085341225", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "e6bd8885e790a8be05bb2a0ed57a46aeb8ba5c8b40829917e5340a1e7d3a9f38" + "run_id": "37204588491", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "BoringCache posthog cold: failure", + "GitHub Actions posthog cold: failure" + ], + "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" } ], "report_state": "current", - "report_path": "results/opentelemetry-java/migration-screening-01/report.json", + "report_path": "results/posthog/shared-combined-lifecycle-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085341225, + "run_id": 37204588491, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 7, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085341225.tar", - "bundle_bytes": 472576, - "sha256": "c5de94bda90ebb24e156895209fc648c5ad7d05cedbd7fb9cae1cd5ec67bff5c", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204588491.tar", + "bundle_bytes": 153088, + "sha256": "cb696ffd068cbf8d95731d03230d730d084c21c43b067ddf516d74e388ef575a", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], @@ -6822,8 +9629,8 @@ }, { "case_id": "posthog", - "series_id": "image-output-combined-qualification-01", - "plan_path": "results/posthog/image-output-combined-qualification-01/series.json", + "series_id": "shared-combined-lifecycle-02", + "plan_path": "results/posthog/shared-combined-lifecycle-02/series.json", "lane": "fresh", "variant": "combined", "sample_count": 1, @@ -6849,16 +9656,13 @@ } ] }, - "plan_sha256": "1cfc86fd12483a24861a1b5e25e6e705b2e057e4ec730b177b2d4bae1b9c36ad", - "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, + "valid_for_comparison": true, "execution_verified": true, - "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [], "summaries": [ { @@ -6866,9 +9670,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 1889, - "min": 1889, - "max": 1889 + "median": 1703, + "min": 1703, + "max": 1703 }, "storage": null, "storage_measured_count": 0 @@ -6878,9 +9682,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 187, - "min": 187, - "max": 187 + "median": 172, + "min": 172, + "max": 172 }, "storage": null, "storage_measured_count": 0 @@ -6890,14 +9694,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 1016, - "min": 1016, - "max": 1016 + "median": 842, + "min": 842, + "max": 842 }, "storage": { - "median": 6237909312, - "min": 6237909312, - "max": 6237909312 + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 }, "storage_measured_count": 1 }, @@ -6906,14 +9710,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 162, - "min": 162, - "max": 162 + "median": 145, + "min": 145, + "max": 145 }, "storage": { - "median": 6237909312, - "min": 6237909312, - "max": 6237909312 + "median": 6242034116, + "min": 6242034116, + "max": 6242034116 }, "storage_measured_count": 1 } @@ -6928,45 +9732,45 @@ "case_id": "posthog", "cli_version": "v1.33.0", "variant": "combined", - "series_id": "image-output-combined-qualification-01", + "series_id": "shared-combined-lifecycle-02", "sample": "1" }, - "ref": "docker-output-correction-2026-10-03", + "ref": "shared-lifecycle-correction-2026-10-04", "state": "requested", - "requested_at": "2026-10-03T19:34:21Z", - "run_id": 37148377350, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350" + "requested_at": "2026-10-04T13:17:50Z", + "run_id": 37205114269, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269" } ], "completions": [ { "schema_version": 1, "case_id": "posthog", - "series_id": "image-output-combined-qualification-01", + "series_id": "shared-combined-lifecycle-02", "sample": 1, - "run_id": "37148377350", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148377350", - "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "run_id": "37205114269", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "7d2cf7f6e9b12c476c9b3e23b73d31fbe8f25176a91bf7d18fc29a027e819895" + "evidence_manifest_sha256": "fb53406ccf8cfb4dabca781b2ce43e740bffb2ba0aee9c24a6af88cb3fc3f8ca" } ], "report_state": "current", - "report_path": "results/posthog/image-output-combined-qualification-01/report.json", + "report_path": "results/posthog/shared-combined-lifecycle-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37148377350, + "run_id": 37205114269, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 15, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148377350.tar", - "bundle_bytes": 373248, - "sha256": "bf89e6b23b5517ca298ff620dde1885fce19ec9a9a8d92fce5092a2dfd93e893", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205114269.tar", + "bundle_bytes": 742912, + "sha256": "0775152becae67e1be4aaf4082be989879111190ddc8aeb66c820bc82a6f3d31", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], @@ -6974,8 +9778,8 @@ }, { "case_id": "posthog", - "series_id": "image-output-qualification-01", - "plan_path": "results/posthog/image-output-qualification-01/series.json", + "series_id": "shared-layers-lifecycle-01", + "plan_path": "results/posthog/shared-layers-lifecycle-01/series.json", "lane": "fresh", "variant": "layers", "sample_count": 1, @@ -7001,19 +9805,24 @@ } ] }, - "plan_sha256": "5bae03dfe4b1b09e3163a65a65a9cc480adf19f61e620dcdff993864b59433f2", - "definition_sha256": "0c34f204ef6467f2d367e5ed76ad8991a3ced222d9e1f5703ea6a21a5fd8a960", + "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44", + "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [ - "Both warm jobs failed before building because legacy published-scope validation ran before native series scope resolution.", - "Cold build_seconds includes the superseded loaded-image inspection, contrary to the declared timing scope. Preserve as diagnostic; corrected execution uses a new series.", - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "actions-cache" + ], [ 1, "warm", @@ -7025,36 +9834,7 @@ "actions-cache" ] ], - "summaries": [ - { - "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 1579, - "min": 1579, - "max": 1579 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "cold", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 813, - "min": 813, - "max": 813 - }, - "storage": { - "median": 6237669195, - "min": 6237669195, - "max": 6237669195 - }, - "storage_measured_count": 1 - } - ], + "summaries": [], "dispatches": [ { "case_id": "posthog", @@ -7065,49 +9845,49 @@ "case_id": "posthog", "cli_version": "v1.33.0", "variant": "layers", - "series_id": "image-output-qualification-01", + "series_id": "shared-layers-lifecycle-01", "sample": "1" }, - "ref": "image-output-qualification-2026-10-03", + "ref": "shared-lifecycle-2026-10-04", "state": "requested", - "requested_at": "2026-10-03T06:54:57Z", - "run_id": 37104600856, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856" + "requested_at": "2026-10-04T13:08:58Z", + "run_id": 37204586535, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535" } ], "completions": [ { "schema_version": 1, "case_id": "posthog", - "series_id": "image-output-qualification-01", + "series_id": "shared-layers-lifecycle-01", "sample": 1, - "run_id": "37104600856", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37104600856", - "source_sha": "35e10981f5e27824d81b19c575a91691592905e9", + "run_id": "37204586535", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", + "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", "github_conclusion": "failure", "status": "failed", "errors": [ "Workflow concluded failure", - "GitHub Actions posthog warm: failure", - "BoringCache posthog warm: failure" + "GitHub Actions posthog cold: failure", + "BoringCache posthog cold: failure" ], - "evidence_manifest_sha256": "61aee72e30eb1e9f7e21dc0bf4c65e62b3d33de24057f5343593919036e6f320" + "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" } ], "report_state": "current", - "report_path": "results/posthog/image-output-qualification-01/report.json", + "report_path": "results/posthog/shared-layers-lifecycle-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37104600856, + "run_id": 37204586535, "inventory_state": "complete", - "verified_files": 9, + "verified_files": 7, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-image-output-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-image-output-2026-10-03/benchmarks-37104600856.tar", - "bundle_bytes": 392704, - "sha256": "6a38f37864533c9921020e3d994a21914adf337775f42b734e74fd894eebc7e2", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204586535.tar", + "bundle_bytes": 153088, + "sha256": "03afa636a012405786868313088132c167e9a5e3b305c0d53761604fce3f3558", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], @@ -7115,8 +9895,8 @@ }, { "case_id": "posthog", - "series_id": "image-output-qualification-02", - "plan_path": "results/posthog/image-output-qualification-02/series.json", + "series_id": "shared-layers-lifecycle-02", + "plan_path": "results/posthog/shared-layers-lifecycle-02/series.json", "lane": "fresh", "variant": "layers", "sample_count": 1, @@ -7142,16 +9922,13 @@ } ] }, - "plan_sha256": "927f06fe53938ebf630a4e155e08ce958db2636f839a15b4f299c1c916f448fc", - "definition_sha256": "6575963f1c26bb20deabd14c54d92bf87354df90a4c8e0171c6674aaaf579644", + "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", + "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": false, + "valid_for_comparison": true, "execution_verified": true, - "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." - ], + "methodology_issues": [], "missing_observations": [], "summaries": [ { @@ -7159,9 +9936,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 1609, - "min": 1609, - "max": 1609 + "median": 1845, + "min": 1845, + "max": 1845 }, "storage": null, "storage_measured_count": 0 @@ -7171,9 +9948,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 175, - "min": 175, - "max": 175 + "median": 226, + "min": 226, + "max": 226 }, "storage": null, "storage_measured_count": 0 @@ -7183,14 +9960,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 778, - "min": 778, - "max": 778 + "median": 953, + "min": 953, + "max": 953 }, "storage": { - "median": 6237262375, - "min": 6237262375, - "max": 6237262375 + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 }, "storage_measured_count": 1 }, @@ -7199,14 +9976,14 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 136, - "min": 136, - "max": 136 + "median": 139, + "min": 139, + "max": 139 }, "storage": { - "median": 6237262375, - "min": 6237262375, - "max": 6237262375 + "median": 6238377174, + "min": 6238377174, + "max": 6238377174 }, "storage_measured_count": 1 } @@ -7221,56 +9998,56 @@ "case_id": "posthog", "cli_version": "v1.33.0", "variant": "layers", - "series_id": "image-output-qualification-02", + "series_id": "shared-layers-lifecycle-02", "sample": "1" }, - "ref": "docker-output-correction-2026-10-03", + "ref": "shared-lifecycle-correction-2026-10-04", "state": "requested", - "requested_at": "2026-10-03T19:34:19Z", - "run_id": 37148375181, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181" + "requested_at": "2026-10-04T13:17:48Z", + "run_id": 37205112800, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800" } ], "completions": [ { "schema_version": 1, "case_id": "posthog", - "series_id": "image-output-qualification-02", + "series_id": "shared-layers-lifecycle-02", "sample": 1, - "run_id": "37148375181", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37148375181", - "source_sha": "ebe9215a904081d279458722339ef85d2b742db6", + "run_id": "37205112800", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800", + "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "ad47456c459688cb932e9d232c57790b64b694616f587246c3bb3b3c07e77e25" + "evidence_manifest_sha256": "7b265e188abeb7d75f9a8be7a820e25d864b76ff43db00e9267f8606bf3f6c18" } ], "report_state": "current", - "report_path": "results/posthog/image-output-qualification-02/report.json", + "report_path": "results/posthog/shared-layers-lifecycle-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37148375181, + "run_id": 37205112800, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 15, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-lifecycle-review-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-lifecycle-review-2026-10-04/benchmarks-37148375181.tar", - "bundle_bytes": 372736, - "sha256": "49df16499aee7a038dfea048f8ec82e605cbfa22fa8822e0060bc7ce213e65b4", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205112800.tar", + "bundle_bytes": 738304, + "sha256": "6206f8fad70e7ef25ca0b49fc4fde438477aa7c633f3e901d7f12c992d89c6ed", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "posthog", - "series_id": "shared-combined-lifecycle-01", - "plan_path": "results/posthog/shared-combined-lifecycle-01/series.json", + "case_id": "qdrant", + "series_id": "migration-screening-01", + "plan_path": "results/qdrant/migration-screening-01/series.json", "lane": "fresh", - "variant": "combined", + "variant": null, "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -7278,116 +10055,150 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "PostHog/posthog", + "repository": "qdrant/qdrant", "pins": [ { "path": "upstream", - "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "revision": "9ae05efdc1ef96352889b2a6b6b574c35232b974", "kind": "gitlink" } ] }, - "plan_sha256": "683444f5363548d3815f9e55d6a56a023d9d15a9300582fe02bfeb322f2da2e7", - "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", - "state": "failed", + "plan_sha256": "56f373789edbd2cc069a8e6feed14f4fca492f1c1381fe86ff3f20152050571f", + "definition_sha256": "e05d45d9cb64f12d4e8b734ca13bc253a3a03c7f2833add246c8981abe668bf7", + "state": "completed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": false, - "methodology_issues": [], - "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], - [ - 1, - "cold", - "actions-cache" - ], - [ - 1, - "warm", - "boringcache" - ], - [ - 1, - "warm", - "actions-cache" - ] + "execution_verified": true, + "methodology_issues": [ + "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", + "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 804, + "min": 804, + "max": 804 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 21, + "min": 21, + "max": 21 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 576, + "min": 576, + "max": 576 + }, + "storage": { + "median": 2084676721, + "min": 2084676721, + "max": 2084676721 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 25, + "min": 25, + "max": 25 + }, + "storage": { + "median": 2084676721, + "min": 2084676721, + "max": 2084676721 + }, + "storage_measured_count": 1 + } ], - "summaries": [], "dispatches": [ { - "case_id": "posthog", + "case_id": "qdrant", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "posthog", + "case_id": "qdrant", "cli_version": "v1.33.0", - "variant": "combined", - "series_id": "shared-combined-lifecycle-01", + "series_id": "migration-screening-01", "sample": "1" }, - "ref": "shared-lifecycle-2026-10-04", + "ref": "active-case-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-04T13:09:00Z", - "run_id": 37204588491, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491" + "requested_at": "2026-10-03T01:14:43Z", + "run_id": 37085343088, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085343088" } ], "completions": [ { "schema_version": 1, - "case_id": "posthog", - "series_id": "shared-combined-lifecycle-01", + "case_id": "qdrant", + "series_id": "migration-screening-01", "sample": 1, - "run_id": "37204588491", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204588491", - "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "BoringCache posthog cold: failure", - "GitHub Actions posthog cold: failure" - ], - "evidence_manifest_sha256": "d60cbda03aeda419341ff8112992d81f32b476fbe508da86851a1f4994d2be7f" + "run_id": "37085343088", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085343088", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "046283d727df85851ca8b4bd6e761be0c03584aba7029f4e7bd4192bd2b86547" } ], "report_state": "current", - "report_path": "results/posthog/shared-combined-lifecycle-01/report.json", + "report_path": "results/qdrant/migration-screening-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37204588491, + "run_id": 37085343088, "inventory_state": "complete", - "verified_files": 7, + "verified_files": 13, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204588491.tar", - "bundle_bytes": 153088, - "sha256": "cb696ffd068cbf8d95731d03230d730d084c21c43b067ddf516d74e388ef575a", + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085343088.tar", + "bundle_bytes": 508928, + "sha256": "d0e8df182c60289a678c10bcbd8cdfe281afe474a335ec883a9b4310c9ec5afb", "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" } ], "runs_without_archives": [] }, { - "case_id": "posthog", - "series_id": "shared-combined-lifecycle-02", - "plan_path": "results/posthog/shared-combined-lifecycle-02/series.json", + "case_id": "spring-ai", + "series_id": "migration-screening-01", + "plan_path": "results/spring-ai/migration-screening-01/series.json", "lane": "fresh", - "variant": "combined", + "variant": null, "sample_count": 1, "comparison": { "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", @@ -7395,24 +10206,24 @@ "boringcache", "actions-cache" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", "primary_metric": "build_and_reuse_seconds", "storage": "provider-reported", "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", "sample_count": 10 }, "source": { - "repository": "PostHog/posthog", + "repository": "spring-projects/spring-ai", "pins": [ { "path": "upstream", - "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", + "revision": "46d05c2fdbab52f6ef04207abe19b12687a47453", "kind": "gitlink" } ] }, - "plan_sha256": "1b354ee844ed380919a4c5c1abf0cdb45d63489c658b706c3d280c68ab5830f2", - "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", + "plan_sha256": "f1c6eb6db912cb6be471ab7649e5cc30d8222754a6055aeb81fd02f87b7768e7", + "definition_sha256": "74bcd4a8848bd296578e9c56f18f5a7d83b33908b2e54f1a9f95f79ea24fd9be", "state": "completed", "publication": "unreviewed", "valid_for_comparison": true, @@ -7425,9 +10236,9 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 1703, - "min": 1703, - "max": 1703 + "median": 514, + "min": 514, + "max": 514 }, "storage": null, "storage_measured_count": 0 @@ -7437,26 +10248,30 @@ "provider": "actions-cache", "count": 1, "measurement": { - "median": 172, - "min": 172, - "max": 172 + "median": 118, + "min": 118, + "max": 118 }, - "storage": null, - "storage_measured_count": 0 + "storage": { + "median": 189388365, + "min": 189388365, + "max": 189388365 + }, + "storage_measured_count": 1 }, { "phase": "cold", "provider": "boringcache", "count": 1, "measurement": { - "median": 842, - "min": 842, - "max": 842 + "median": 519, + "min": 519, + "max": 519 }, "storage": { - "median": 6242034116, - "min": 6242034116, - "max": 6242034116 + "median": 207205552, + "min": 207205552, + "max": 207205552 }, "storage_measured_count": 1 }, @@ -7465,108 +10280,280 @@ "provider": "boringcache", "count": 1, "measurement": { - "median": 145, - "min": 145, - "max": 145 + "median": 83, + "min": 83, + "max": 83 }, "storage": { - "median": 6242034116, - "min": 6242034116, - "max": 6242034116 + "median": 210696542, + "min": 210696542, + "max": 210696542 }, "storage_measured_count": 1 } ], "dispatches": [ { - "case_id": "posthog", + "case_id": "spring-ai", "repository": "boringcache/benchmarks", "workflow": "native-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "posthog", + "case_id": "spring-ai", "cli_version": "v1.33.0", - "variant": "combined", - "series_id": "shared-combined-lifecycle-02", + "series_id": "migration-screening-01", "sample": "1" }, - "ref": "shared-lifecycle-correction-2026-10-04", + "ref": "active-case-qualification-2026-10-03", "state": "requested", - "requested_at": "2026-10-04T13:17:50Z", - "run_id": 37205114269, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269" + "requested_at": "2026-10-03T01:14:45Z", + "run_id": 37085344978, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085344978" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "spring-ai", + "series_id": "migration-screening-01", + "sample": 1, + "run_id": "37085344978", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085344978", + "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "3920c72a124f43895cc37e27f4821bfd943f0f2e8d0433d94459383dad97092e" + } + ], + "report_state": "current", + "report_path": "results/spring-ai/migration-screening-01/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37085344978, + "inventory_state": "complete", + "verified_files": 15, + "gaps": [], + "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-active-completion-2026-10-03", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-active-completion-2026-10-03/benchmarks-37085344978.tar", + "bundle_bytes": 749568, + "sha256": "92f61ac07e77a9c656e4389158a3281012175e4e43857965b383e392a7519829", + "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "plan_path": "results/stackstorm-pants/registry-qualification-01/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 2, + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "plan_sha256": "ae3ee8ca9d1fcd7b05d186e12a52dd03b42eebaf92f9480c27ea9bd632ac8a58", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "state": "completed", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": true, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "dispatches": [ + { + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "1" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:33:11Z", + "run_id": 37303666267, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + }, + { + "case_id": "stackstorm-pants", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:33:12Z", + "run_id": 37303670119, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303670119" } ], "completions": [ { "schema_version": 1, - "case_id": "posthog", - "series_id": "shared-combined-lifecycle-02", - "sample": 1, - "run_id": "37205114269", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205114269", - "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37303666267", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303666267", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2793052a56437ecb28302e51716a9cd954392e5328c64304e6f1bc3171eb4d32" + }, + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37303670119", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303670119", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "fb53406ccf8cfb4dabca781b2ce43e740bffb2ba0aee9c24a6af88cb3fc3f8ca" + "evidence_manifest_sha256": "314c3c10a84c6538172ddcd5977feda6592c73d39bc987efa4250cd1757f7808" } ], "report_state": "current", - "report_path": "results/posthog/shared-combined-lifecycle-02/report.json", + "report_path": "results/stackstorm-pants/registry-qualification-01/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37205114269, + "run_id": 37303666267, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205114269.tar", - "bundle_bytes": 742912, - "sha256": "0775152becae67e1be4aaf4082be989879111190ddc8aeb66c820bc82a6f3d31", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303666267.tar.gz", + "bundle_bytes": 65356159, + "sha256": "79bb4a1106d25013ebf82f52b9568780b977d79453af61a0007e01d8f9864e77", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37303670119, + "inventory_state": "complete", + "verified_files": 16, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303670119.tar.gz", + "bundle_bytes": 65356964, + "sha256": "4e31b69eea024ab115e557d5f207bac45f739082332934485e30783548fcc83e", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "posthog", - "series_id": "shared-layers-lifecycle-01", - "plan_path": "results/posthog/shared-layers-lifecycle-01/series.json", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-01", + "plan_path": "results/stackstorm-pants/registry-screening-01/series.json", "lane": "fresh", - "variant": "layers", + "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "PostHog/posthog", + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", "pins": [ { - "path": "upstream", - "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" } ] }, - "plan_sha256": "13c3bd7351df48d08972ba6d0c33f266205703b66a417ce6b3b96d8de674ae44", - "definition_sha256": "6d012067d6265b02726f6c5df471c31178c21c92c4471ce712ae7a3d4125f653", - "state": "failed", + "plan_sha256": "940699d14bf7b7b7b27491b381437f2d9fc6571b83ee35108b4cd382259d48f0", + "definition_sha256": "c33c35df34ae6ddf04d9b337d377461b155873c879d93f4fdc4a7b1722cd22d2", + "state": "planned", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [ [ 1, @@ -7576,7 +10563,7 @@ [ 1, "cold", - "actions-cache" + "bazel-remote" ], [ 1, @@ -7586,516 +10573,378 @@ [ 1, "warm", - "actions-cache" + "bazel-remote" ] ], "summaries": [], - "dispatches": [ - { - "case_id": "posthog", - "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", - "lane": "fresh", - "inputs": { - "case_id": "posthog", - "cli_version": "v1.33.0", - "variant": "layers", - "series_id": "shared-layers-lifecycle-01", - "sample": "1" - }, - "ref": "shared-lifecycle-2026-10-04", - "state": "requested", - "requested_at": "2026-10-04T13:08:58Z", - "run_id": 37204586535, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535" - } - ], - "completions": [ - { - "schema_version": 1, - "case_id": "posthog", - "series_id": "shared-layers-lifecycle-01", - "sample": 1, - "run_id": "37204586535", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37204586535", - "source_sha": "f6bcde907231d8e70c34d05e522e393db799713d", - "github_conclusion": "failure", - "status": "failed", - "errors": [ - "Workflow concluded failure", - "GitHub Actions posthog cold: failure", - "BoringCache posthog cold: failure" - ], - "evidence_manifest_sha256": "05001c070314e7cfead2f2e271c9af2fda780684c6a7ce1ff30d49b38db2e70a" - } - ], - "report_state": "current", - "report_path": "results/posthog/shared-layers-lifecycle-01/report.json", - "evidence": [ - { - "repository": "boringcache/benchmarks", - "run_id": 37204586535, - "inventory_state": "complete", - "verified_files": 7, - "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37204586535.tar", - "bundle_bytes": 153088, - "sha256": "03afa636a012405786868313088132c167e9a5e3b305c0d53761604fce3f3558", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" - } - ], + "dispatches": [], + "completions": [], + "report_state": "missing", + "report_path": null, + "evidence": [], "runs_without_archives": [] }, { - "case_id": "posthog", - "series_id": "shared-layers-lifecycle-02", - "plan_path": "results/posthog/shared-layers-lifecycle-02/series.json", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-02", + "plan_path": "results/stackstorm-pants/registry-screening-02/series.json", "lane": "fresh", - "variant": "layers", + "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed Docker build including image export and load, plus measured provider setup; output inspection is outside the timed build; provider post-job work is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "PostHog/posthog", + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", "pins": [ { - "path": "upstream", - "revision": "662f459147d2df2a44b008e4a9b6e75d8ac46e93", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" } ] }, - "plan_sha256": "9440f1ecc87acfc5f42121a9b72a446881769df6f188c1a27f50070c209dd0d1", - "definition_sha256": "5ed26c35a502999eb61efd535b39249af12b0d86f1694be37b0d020c4202178d", - "state": "completed", + "plan_sha256": "9fd0981ad3a1f8c068ad8be0ca85a799d4eef6d67e94e2bbf5851e9cce9662e6", + "definition_sha256": "6717b361fea80c5cd76c17181a11ced0dda003db1886b353840c494962e2f3a4", + "state": "failed", "publication": "unreviewed", - "valid_for_comparison": true, - "execution_verified": true, - "methodology_issues": [], - "missing_observations": [], - "summaries": [ - { - "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 1845, - "min": 1845, - "max": 1845 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 226, - "min": 226, - "max": 226 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "cold", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 953, - "min": 953, - "max": 953 - }, - "storage": { - "median": 6238377174, - "min": 6238377174, - "max": 6238377174 - }, - "storage_measured_count": 1 - }, - { - "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 139, - "min": 139, - "max": 139 - }, - "storage": { - "median": 6238377174, - "min": 6238377174, - "max": 6238377174 - }, - "storage_measured_count": 1 - } + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] ], + "summaries": [], "dispatches": [ { - "case_id": "posthog", + "case_id": "stackstorm-pants", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "posthog", - "cli_version": "v1.33.0", - "variant": "layers", - "series_id": "shared-layers-lifecycle-02", + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-02", "sample": "1" }, - "ref": "shared-lifecycle-correction-2026-10-04", + "ref": "main", "state": "requested", - "requested_at": "2026-10-04T13:17:48Z", - "run_id": 37205112800, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800" + "requested_at": "2026-10-05T11:17:36Z", + "run_id": 37302004103, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103" } ], "completions": [ { "schema_version": 1, - "case_id": "posthog", - "series_id": "shared-layers-lifecycle-02", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-02", "sample": 1, - "run_id": "37205112800", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37205112800", - "source_sha": "d045da272fae61c96d06727459055f1a31eca48f", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "7b265e188abeb7d75f9a8be7a820e25d864b76ff43db00e9267f8606bf3f6c18" + "run_id": "37302004103", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "ede5e77e9b757f31622697cec3fa93f8667cb0fd0d6d2cf1e5de44c94b3a8b35" } ], "report_state": "current", - "report_path": "results/posthog/shared-layers-lifecycle-02/report.json", + "report_path": "results/stackstorm-pants/registry-screening-02/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37205112800, + "run_id": 37302004103, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 9, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not an audit of an original repository", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-shared-lifecycle-2026-10-04", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-shared-lifecycle-2026-10-04/benchmarks-37205112800.tar", - "bundle_bytes": 738304, - "sha256": "6206f8fad70e7ef25ca0b49fc4fde438477aa7c633f3e901d7f12c992d89c6ed", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302004103.tar.gz", + "bundle_bytes": 91046, + "sha256": "cce453e95a4939a69a4065301006fb80358337029f10c042aee818b26bfb667b", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "qdrant", - "series_id": "migration-screening-01", - "plan_path": "results/qdrant/migration-screening-01/series.json", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", + "plan_path": "results/stackstorm-pants/registry-screening-03/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "qdrant/qdrant", + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", "pins": [ { - "path": "upstream", - "revision": "9ae05efdc1ef96352889b2a6b6b574c35232b974", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" } ] }, - "plan_sha256": "56f373789edbd2cc069a8e6feed14f4fca492f1c1381fe86ff3f20152050571f", - "definition_sha256": "e05d45d9cb64f12d4e8b734ca13bc253a3a03c7f2833add246c8981abe668bf7", - "state": "completed", + "plan_sha256": "7cd6568e943d76372434e5ba184fd8204575671dc220d2c042b3e6cbd9200034", + "definition_sha256": "72cc5c0680d703db46455d69fda311472e215b8423b3ae95b8276f4c8b040e4f", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, - "execution_verified": true, + "execution_verified": false, "methodology_issues": [ - "The execution definition exported Actions Cache state on warm replay while BoringCache used restore-only trust. These observations do not compare the same publication lifecycle.", - "The workflow retained derived phase records but did not retain the original final BoringCache Action evidence JSON after cleanup." + "This series does not declare a comparison of provider performance." ], - "missing_observations": [], - "summaries": [ - { - "phase": "cold", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 804, - "min": 804, - "max": 804 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "warm", - "provider": "actions-cache", - "count": 1, - "measurement": { - "median": 21, - "min": 21, - "max": 21 - }, - "storage": null, - "storage_measured_count": 0 - }, - { - "phase": "cold", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 576, - "min": 576, - "max": 576 - }, - "storage": { - "median": 2084676721, - "min": 2084676721, - "max": 2084676721 - }, - "storage_measured_count": 1 - }, - { - "phase": "warm", - "provider": "boringcache", - "count": 1, - "measurement": { - "median": 25, - "min": 25, - "max": 25 - }, - "storage": { - "median": 2084676721, - "min": 2084676721, - "max": 2084676721 - }, - "storage_measured_count": 1 - } + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] ], + "summaries": [], "dispatches": [ { - "case_id": "qdrant", + "case_id": "stackstorm-pants", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "qdrant", - "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-03", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-03T01:14:43Z", - "run_id": 37085343088, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085343088" + "requested_at": "2026-10-05T11:20:48Z", + "run_id": 37302345686, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686" } ], "completions": [ { "schema_version": 1, - "case_id": "qdrant", - "series_id": "migration-screening-01", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", "sample": 1, - "run_id": "37085343088", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085343088", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", - "github_conclusion": "success", - "status": "success", - "errors": [], - "evidence_manifest_sha256": "046283d727df85851ca8b4bd6e761be0c03584aba7029f4e7bd4192bd2b86547" + "run_id": "37302345686", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686", + "source_sha": "6aebc8f9f33a9e8de209e747b48c0a4f2fdda60a", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "84e6a78d443c929f68096abc6c210d3a6783cfdd3f5fd444bed40c34c7b957cf" } ], "report_state": "current", - "report_path": "results/qdrant/migration-screening-01/report.json", + "report_path": "results/stackstorm-pants/registry-screening-03/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085343088, + "run_id": 37302345686, "inventory_state": "complete", - "verified_files": 13, + "verified_files": 9, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/active-case-qualification-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/active-case-qualification-2026-10-03/benchmarks-37085343088.tar", - "bundle_bytes": 508928, - "sha256": "d0e8df182c60289a678c10bcbd8cdfe281afe474a335ec883a9b4310c9ec5afb", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302345686.tar.gz", + "bundle_bytes": 67274, + "sha256": "d2f7b85adc90f4291d0faad94607dbd3b17eb18f029f26f39c8a546995821851", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] }, { - "case_id": "spring-ai", - "series_id": "migration-screening-01", - "plan_path": "results/spring-ai/migration-screening-01/series.json", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-04", + "plan_path": "results/stackstorm-pants/registry-screening-04/series.json", "lane": "fresh", "variant": null, "sample_count": 1, "comparison": { - "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", "providers": [ "boringcache", - "actions-cache" + "bazel-remote" ], - "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", - "primary_metric": "build_and_reuse_seconds", - "storage": "provider-reported", - "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", - "sample_count": 10 + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 }, "source": { - "repository": "spring-projects/spring-ai", + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", "pins": [ { - "path": "upstream", - "revision": "46d05c2fdbab52f6ef04207abe19b12687a47453", - "kind": "gitlink" + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" } ] }, - "plan_sha256": "f1c6eb6db912cb6be471ab7649e5cc30d8222754a6055aeb81fd02f87b7768e7", - "definition_sha256": "74bcd4a8848bd296578e9c56f18f5a7d83b33908b2e54f1a9f95f79ea24fd9be", + "plan_sha256": "e085a1693be2da021182976715305294e9d52fdc467d0e5f4b3b174b1a78d1d7", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", "state": "completed", "publication": "unreviewed", - "valid_for_comparison": true, + "valid_for_comparison": false, "execution_verified": true, - "methodology_issues": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], "missing_observations": [], "summaries": [ { "phase": "cold", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 514, - "min": 514, - "max": 514 - }, + "measurement": null, "storage": null, "storage_measured_count": 0 }, { "phase": "warm", - "provider": "actions-cache", + "provider": "bazel-remote", "count": 1, - "measurement": { - "median": 118, - "min": 118, - "max": 118 - }, - "storage": { - "median": 189388365, - "min": 189388365, - "max": 189388365 - }, - "storage_measured_count": 1 + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "cold", "provider": "boringcache", "count": 1, - "measurement": { - "median": 519, - "min": 519, - "max": 519 - }, - "storage": { - "median": 207205552, - "min": 207205552, - "max": 207205552 - }, - "storage_measured_count": 1 + "measurement": null, + "storage": null, + "storage_measured_count": 0 }, { "phase": "warm", "provider": "boringcache", "count": 1, - "measurement": { - "median": 83, - "min": 83, - "max": 83 - }, - "storage": { - "median": 210696542, - "min": 210696542, - "max": 210696542 - }, - "storage_measured_count": 1 + "measurement": null, + "storage": null, + "storage_measured_count": 0 } ], "dispatches": [ { - "case_id": "spring-ai", + "case_id": "stackstorm-pants", "repository": "boringcache/benchmarks", - "workflow": "native-fresh-benchmark.yml", + "workflow": "reapi-fresh-benchmark.yml", "lane": "fresh", "inputs": { - "case_id": "spring-ai", - "cli_version": "v1.33.0", - "series_id": "migration-screening-01", + "case_id": "stackstorm-pants", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-04", "sample": "1" }, - "ref": "active-case-qualification-2026-10-03", + "ref": "qualify-native-registry", "state": "requested", - "requested_at": "2026-10-03T01:14:45Z", - "run_id": 37085344978, - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085344978" + "requested_at": "2026-10-05T11:26:00Z", + "run_id": 37302896058, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302896058" } ], "completions": [ { "schema_version": 1, - "case_id": "spring-ai", - "series_id": "migration-screening-01", + "case_id": "stackstorm-pants", + "series_id": "registry-screening-04", "sample": 1, - "run_id": "37085344978", - "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37085344978", - "source_sha": "da6400b319991a6b5e36b1e2fc70eaca2619a560", + "run_id": "37302896058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302896058", + "source_sha": "d63d3593c150f7471c7419a5bac6827c2979c3ef", "github_conclusion": "success", "status": "success", "errors": [], - "evidence_manifest_sha256": "3920c72a124f43895cc37e27f4821bfd943f0f2e8d0433d94459383dad97092e" + "evidence_manifest_sha256": "d9b28308dd0b7ffbc7d745fd577148d8ec4449eacad97cdfa729ecb399e905ed" } ], "report_state": "current", - "report_path": "results/spring-ai/migration-screening-01/report.json", + "report_path": "results/stackstorm-pants/registry-screening-04/report.json", "evidence": [ { "repository": "boringcache/benchmarks", - "run_id": 37085344978, + "run_id": 37302896058, "inventory_state": "complete", - "verified_files": 15, + "verified_files": 16, "gaps": [], - "scope": "This run's available attempts, jobs, logs, artifacts, commit metadata, and workflow; not a whole-repository audit", - "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-active-completion-2026-10-03", - "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-active-completion-2026-10-03/benchmarks-37085344978.tar", - "bundle_bytes": 749568, - "sha256": "92f61ac07e77a9c656e4389158a3281012175e4e43857965b383e392a7519829", - "publication_verification": "Downloaded uploaded assets, verified archive SHA-256, extracted bundle, and verified all listed run files and inventory" + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302896058.tar.gz", + "bundle_bytes": 65354461, + "sha256": "172a55594cb2938563cc67a8e4b92a2ba26da77d6f674ce14a974fce74c2a17a", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" } ], "runs_without_archives": [] diff --git a/migration/evidence.json b/migration/evidence.json index ac5786910..8e5046742 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -1287,6 +1287,258 @@ "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", "verified_on": "2026-10-05", "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37301997879, + "verified_files": 16, + "gaps": [], + "sha256": "56088bc5bf76d8a296beca3cd0ceac2265dc449bd0baf807ddad532a92d21767", + "bundle_bytes": 3628592, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37301997879.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302001164, + "verified_files": 15, + "gaps": [], + "sha256": "64aeb093857a8b1dcab08060ddf79c4ce8b85af3f39579141071ea91cfc42e7c", + "bundle_bytes": 841387, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302001164.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302004103, + "verified_files": 9, + "gaps": [], + "sha256": "cce453e95a4939a69a4065301006fb80358337029f10c042aee818b26bfb667b", + "bundle_bytes": 91046, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302004103.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302010045, + "verified_files": 16, + "gaps": [], + "sha256": "1eaef318a5107d74bf9fa5109707ac16adb2a88ac0996b97961eebbe6b6c3ef5", + "bundle_bytes": 1659064, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302010045.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302007157, + "verified_files": 9, + "gaps": [], + "sha256": "08b74d110cb201e2f8cefd83be941d7e3b2e63ea1cce174edacebff156e2ffcd", + "bundle_bytes": 122488, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302007157.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302345686, + "verified_files": 9, + "gaps": [], + "sha256": "d2f7b85adc90f4291d0faad94607dbd3b17eb18f029f26f39c8a546995821851", + "bundle_bytes": 67274, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302345686.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302476504, + "verified_files": 16, + "gaps": [], + "sha256": "6860e07a5f7ead5a10198483d4c4f9535ee54f54f4041bb6681a87a701b136f6", + "bundle_bytes": 822268, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302476504.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302711581, + "verified_files": 16, + "gaps": [], + "sha256": "a1fff6e0704dbc719fc6f8e54d406828015b7226f51348fc6d8fb5e587be734a", + "bundle_bytes": 3605287, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302711581.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302715178, + "verified_files": 16, + "gaps": [], + "sha256": "361ffdc1d10179e1747213f0b8f17ae4a6d969b86aa1131357131c02f6fde4ae", + "bundle_bytes": 3587778, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302715178.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302718511, + "verified_files": 16, + "gaps": [], + "sha256": "5cb9069086bdd60cd6064a68e4587b3ce20d9b62b14f3db266c8bbdeda51188c", + "bundle_bytes": 1646194, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302718511.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302721578, + "verified_files": 16, + "gaps": [], + "sha256": "77e5498f62a2ff1fe6bcb09728d423f5c60309e1c21e524941b4261d4ff733e7", + "bundle_bytes": 1648859, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302721578.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302896058, + "verified_files": 16, + "gaps": [], + "sha256": "172a55594cb2938563cc67a8e4b92a2ba26da77d6f674ce14a974fce74c2a17a", + "bundle_bytes": 65354461, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302896058.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302678892, + "verified_files": 10, + "gaps": [], + "sha256": "478596b3419d210ba53738011b8decda9402aff7ab897415be11aa7e11b65629", + "bundle_bytes": 106242, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302678892.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302929873, + "verified_files": 16, + "gaps": [], + "sha256": "3231e868af54cc889150817a79d4899af7f3ea064ba4407d061a92fd139ff945", + "bundle_bytes": 824311, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302929873.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37302932865, + "verified_files": 16, + "gaps": [], + "sha256": "c86bc6a965966dc545dffb405efff441a57143d0ef795e75f05543cdbdcf3093", + "bundle_bytes": 824072, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37302932865.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37296157157, + "verified_files": 18, + "gaps": [], + "sha256": "89cb0aded5b7856075555104bd6e276efdb391534590b13201cf612fd1b8b44b", + "bundle_bytes": 2492084, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37296157157.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37303666267, + "verified_files": 16, + "gaps": [], + "sha256": "79bb4a1106d25013ebf82f52b9568780b977d79453af61a0007e01d8f9864e77", + "bundle_bytes": 65356159, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303666267.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37303670119, + "verified_files": 16, + "gaps": [], + "sha256": "4e31b69eea024ab115e557d5f207bac45f739082332934485e30783548fcc83e", + "bundle_bytes": 65356964, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303670119.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" } ] } diff --git a/results/executorch-buck2/registry-screening-02/completions/37302007157.json b/results/executorch-buck2/registry-screening-02/completions/37302007157.json new file mode 100644 index 000000000..919b8fd25 --- /dev/null +++ b/results/executorch-buck2/registry-screening-02/completions/37302007157.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302007157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "475ddd9b7f53e3a6b7ced765ccacb53fe1bdc2d46d2001bd7b28c4c4c94f8cc0" +} diff --git a/results/executorch-buck2/registry-screening-02/report.json b/results/executorch-buck2/registry-screening-02/report.json new file mode 100644 index 000000000..5866b80f5 --- /dev/null +++ b/results/executorch-buck2/registry-screening-02/report.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "plan_sha256": "a5eda8a465657d35135c8cfb6188a55cffb58a62c7b207ce9e0ae265529dd667", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302007157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "475ddd9b7f53e3a6b7ced765ccacb53fe1bdc2d46d2001bd7b28c4c4c94f8cc0" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302007157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302007157", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "475ddd9b7f53e3a6b7ced765ccacb53fe1bdc2d46d2001bd7b28c4c4c94f8cc0" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/executorch-buck2/registry-screening-02/report.md b/results/executorch-buck2/registry-screening-02/report.md new file mode 100644 index 000000000..587576867 --- /dev/null +++ b/results/executorch-buck2/registry-screening-02/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: registry-screening-02 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 0/4 recorded; 0 failed; 4 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37302007157 failed completion checks: + +- Workflow concluded failure +- bazel-remote executorch-buck2 cold: failure +- boringcache executorch-buck2 cold: failure + +Missing observations: + +- Sample 1, BoringCache, Cold build +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/registry-screening-03/completions/37302678892.json b/results/executorch-buck2/registry-screening-03/completions/37302678892.json new file mode 100644 index 000000000..57689e626 --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/completions/37302678892.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302678892", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892", + "source_sha": "829fa130f334a8af9df0c3312027921e2f2caaec", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "fda433158d3ad68c400a99c3b8a7e3f6defa9bcca370e4625e907d34853ecc9b" +} diff --git a/results/executorch-buck2/registry-screening-03/report.json b/results/executorch-buck2/registry-screening-03/report.json new file mode 100644 index 000000000..4c2ed938d --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/report.json @@ -0,0 +1,222 @@ +{ + "schema_version": 1, + "plan_sha256": "2ded7301f2b56ac93494c09c0bb722d6c01766e1528c760f6af15cbce49b25bf", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302678892", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892", + "source_sha": "829fa130f334a8af9df0c3312027921e2f2caaec", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "fda433158d3ad68c400a99c3b8a7e3f6defa9bcca370e4625e907d34853ecc9b" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302678892", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302678892", + "source_sha": "829fa130f334a8af9df0c3312027921e2f2caaec", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "fda433158d3ad68c400a99c3b8a7e3f6defa9bcca370e4625e907d34853ecc9b" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "29364c9b6c1bdf023476a589a9b69a641cee3d1264058927f4e8d8ddd7592ed9", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20137542499998062, + "build_seconds": 154.453031812, + "workflow_seconds": null, + "total_seconds": 154.654407237, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.22982105199997704, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 154.654407237 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-series-registry-screening-03-s1-r37302678892-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302678892", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173993", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302678892-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302678892" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/registry-screening-03/report.md b/results/executorch-buck2/registry-screening-03/report.md new file mode 100644 index 000000000..ad74d4c35 --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: registry-screening-03 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 1/4 recorded; 0 failed; 3 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37302678892 failed completion checks: + +- Workflow concluded failure +- bazel-remote executorch-buck2 cold: failure + +Missing observations: + +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 0.20137542499998062 | 154.453031812 | 154.654407237 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/registry-screening-03/runs/1-boringcache-cold.json b/results/executorch-buck2/registry-screening-03/runs/1-boringcache-cold.json new file mode 100644 index 000000000..16b30c0c4 --- /dev/null +++ b/results/executorch-buck2/registry-screening-03/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "29364c9b6c1bdf023476a589a9b69a641cee3d1264058927f4e8d8ddd7592ed9", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ] + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20137542499998062, + "build_seconds": 154.453031812, + "workflow_seconds": null, + "total_seconds": 154.654407237, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.22982105199997704, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 154.654407237 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-series-registry-screening-03-s1-r37302678892-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302678892", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173993", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302678892-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302678892" + ] +} diff --git a/results/executorch-buck2/registry-screening-05/dispatches/1-reapi-fresh-benchmark.yml.json b/results/executorch-buck2/registry-screening-05/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..22e6f5b2e --- /dev/null +++ b/results/executorch-buck2/registry-screening-05/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-05", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:39:39Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37304358750, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750" +} diff --git a/results/gogs-moon/registry-qualification-01/completions/37302711581.json b/results/gogs-moon/registry-qualification-01/completions/37302711581.json new file mode 100644 index 000000000..c958766b2 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/completions/37302711581.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302711581", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302711581", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2481184fb7a464c4746b379d0038961240a6463104da7cf3b73c107adc0e27a5" +} diff --git a/results/gogs-moon/registry-qualification-01/completions/37302715178.json b/results/gogs-moon/registry-qualification-01/completions/37302715178.json new file mode 100644 index 000000000..e9c12f54b --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/completions/37302715178.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302715178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302715178", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "dfe4c6cef48e640a30e0e08613297ccb7201968816fbccad860d0c9f22e117c7" +} diff --git a/results/gogs-moon/registry-qualification-01/report.json b/results/gogs-moon/registry-qualification-01/report.json new file mode 100644 index 000000000..ddbf4b67d --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/report.json @@ -0,0 +1,1202 @@ +{ + "schema_version": 1, + "plan_sha256": "6f0af9bb2ddd8c15c7be1a0810617d8973841bcd8f8ff77dc61ef299041cd5c5", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302711581", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302711581", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2481184fb7a464c4746b379d0038961240a6463104da7cf3b73c107adc0e27a5" + }, + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302715178", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302715178", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "dfe4c6cef48e640a30e0e08613297ccb7201968816fbccad860d0c9f22e117c7" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107650799999846, + "build_seconds": 9.486168882000001, + "workflow_seconds": null, + "total_seconds": 9.58724539, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004957766000003971, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 9.58724539 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174007", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10098372400000244, + "build_seconds": 1.5746930329999884, + "workflow_seconds": null, + "total_seconds": 1.6756767569999909, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0033243450000099983, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6756767569999909 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174018", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1007511629999982, + "build_seconds": 5.257549611000002, + "workflow_seconds": null, + "total_seconds": 5.358300774, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.2575738129999934, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 5.358300774 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174006", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086347799997952, + "build_seconds": 2.0576024240000095, + "workflow_seconds": null, + "total_seconds": 2.158465901999989, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.23100085199999398, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 2.158465901999989 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174019", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103610000000174, + "build_seconds": 8.49131564599999, + "workflow_seconds": null, + "total_seconds": 8.592351745999991, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.002152588999990712, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.592351745999991 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174010", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011681820000021, + "build_seconds": 1.5704553409999917, + "workflow_seconds": null, + "total_seconds": 1.6716235229999938, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003947570999997652, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6716235229999938 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174020", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20131430900005398, + "build_seconds": 6.354287885000076, + "workflow_seconds": null, + "total_seconds": 6.55560219400013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1709647049999603, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 6.55560219400013 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174009", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3019010590000022, + "build_seconds": 2.263736571999999, + "workflow_seconds": null, + "total_seconds": 2.5656376310000013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.42554220099999895, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 2.5656376310000013 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174021", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/registry-qualification-01/report.md b/results/gogs-moon/registry-qualification-01/report.md new file mode 100644 index 000000000..60f1c9178 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/report.md @@ -0,0 +1,37 @@ +# gogs-moon: registry-qualification-01 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 2 | unmeasured | 0 | +| Warm build | bazel-remote | 2 | unmeasured | 0 | +| Cold build | BoringCache | 2 | unmeasured | 0 | +| Warm build | BoringCache | 2 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10107650799999846 | 9.486168882000001 | 9.58724539 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10098372400000244 | 1.5746930329999884 | 1.6756767569999909 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.1007511629999982 | 5.257549611000002 | 5.358300774 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.10086347799997952 | 2.0576024240000095 | 2.158465901999989 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | +| 2 | bazel-remote | Cold build | 0.10103610000000174 | 8.49131564599999 | 8.592351745999991 | unmeasured | unmeasured | miss | [JSON](runs/2-bazel-remote-cold.json) | +| 2 | bazel-remote | Warm build | 0.1011681820000021 | 1.5704553409999917 | 1.6716235229999938 | unmeasured | unmeasured | hit | [JSON](runs/2-bazel-remote-warm.json) | +| 2 | BoringCache | Cold build | 0.20131430900005398 | 6.354287885000076 | 6.55560219400013 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 0.3019010590000022 | 2.263736571999999 | 2.5656376310000013 | unmeasured | unmeasured | hit | [JSON](runs/2-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-cold.json b/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..cced3c47c --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107650799999846, + "build_seconds": 9.486168882000001, + "workflow_seconds": null, + "total_seconds": 9.58724539, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004957766000003971, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 9.58724539 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174007", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-warm.json b/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..399f00280 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10098372400000244, + "build_seconds": 1.5746930329999884, + "workflow_seconds": null, + "total_seconds": 1.6756767569999909, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0033243450000099983, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6756767569999909 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174018", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/1-boringcache-cold.json b/results/gogs-moon/registry-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..749407cbb --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1007511629999982, + "build_seconds": 5.257549611000002, + "workflow_seconds": null, + "total_seconds": 5.358300774, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.2575738129999934, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 5.358300774 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174006", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/1-boringcache-warm.json b/results/gogs-moon/registry-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..6c8220463 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10086347799997952, + "build_seconds": 2.0576024240000095, + "workflow_seconds": null, + "total_seconds": 2.158465901999989, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.23100085199999398, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 2.158465901999989 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s1-r37302711581-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302711581", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174019", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302711581-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302711581" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-cold.json b/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-cold.json new file mode 100644 index 000000000..0aa57c47e --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103610000000174, + "build_seconds": 8.49131564599999, + "workflow_seconds": null, + "total_seconds": 8.592351745999991, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.002152588999990712, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.592351745999991 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174010", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-warm.json b/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-warm.json new file mode 100644 index 000000000..0f8b72ab7 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/2-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011681820000021, + "build_seconds": 1.5704553409999917, + "workflow_seconds": null, + "total_seconds": 1.6716235229999938, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003947570999997652, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6716235229999938 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174020", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/2-boringcache-cold.json b/results/gogs-moon/registry-qualification-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..8243b8faa --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/2-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20131430900005398, + "build_seconds": 6.354287885000076, + "workflow_seconds": null, + "total_seconds": 6.55560219400013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.1709647049999603, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 6.55560219400013 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174009", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] +} diff --git a/results/gogs-moon/registry-qualification-01/runs/2-boringcache-warm.json b/results/gogs-moon/registry-qualification-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..e09feea63 --- /dev/null +++ b/results/gogs-moon/registry-qualification-01/runs/2-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.3019010590000022, + "build_seconds": 2.263736571999999, + "workflow_seconds": null, + "total_seconds": 2.5656376310000013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.42554220099999895, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 2.5656376310000013 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-qualification-01-s2-r37302715178-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302715178", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174021", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302715178-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302715178" + ] +} diff --git a/results/gogs-moon/registry-screening-02/completions/37301997879.json b/results/gogs-moon/registry-screening-02/completions/37301997879.json new file mode 100644 index 000000000..949749267 --- /dev/null +++ b/results/gogs-moon/registry-screening-02/completions/37301997879.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37301997879", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37301997879", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6590381ed0e9bde4d8e3a5cbf8d4a1efd80ec5dba9fa4689fe64a0b3f886bbe6" +} diff --git a/results/gogs-moon/registry-screening-02/report.json b/results/gogs-moon/registry-screening-02/report.json new file mode 100644 index 000000000..47ea652da --- /dev/null +++ b/results/gogs-moon/registry-screening-02/report.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "plan_sha256": "470d098106a74bafaa7ac215832052f8406a2d8c86b298be4fbda5b9bc416a9c", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "gogs-moon", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37301997879", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37301997879", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "6590381ed0e9bde4d8e3a5cbf8d4a1efd80ec5dba9fa4689fe64a0b3f886bbe6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102896799998007, + "build_seconds": 9.582570386000015, + "workflow_seconds": null, + "total_seconds": 9.683599353999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004706392000002779, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 9.683599353999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173909", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111357099998486, + "build_seconds": 1.5931755330000215, + "workflow_seconds": null, + "total_seconds": 1.6942891040000063, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003638071000011678, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6942891040000063 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173921", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20126020800000788, + "build_seconds": 7.471596422999994, + "workflow_seconds": null, + "total_seconds": 7.672856631000002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.0447996050000086, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 7.672856631000002 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173910", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] + }, + { + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20114219899998886, + "build_seconds": 3.7727488040000026, + "workflow_seconds": null, + "total_seconds": 3.9738910029999914, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3115027010000091, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 3.9738910029999914 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173920", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] + } + ], + "exclusions": [] +} diff --git a/results/gogs-moon/registry-screening-02/report.md b/results/gogs-moon/registry-screening-02/report.md new file mode 100644 index 000000000..d8db66d76 --- /dev/null +++ b/results/gogs-moon/registry-screening-02/report.md @@ -0,0 +1,33 @@ +# gogs-moon: registry-screening-02 + +Question: Does Moon restore the Gogs web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 4/4 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 1 | unmeasured | 0 | +| Warm build | bazel-remote | 1 | unmeasured | 0 | +| Cold build | BoringCache | 1 | unmeasured | 0 | +| Warm build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10102896799998007 | 9.582570386000015 | 9.683599353999995 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10111357099998486 | 1.5931755330000215 | 1.6942891040000063 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.20126020800000788 | 7.471596422999994 | 7.672856631000002 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.20114219899998886 | 3.7727488040000026 | 3.9738910029999914 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-cold.json b/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..166740b23 --- /dev/null +++ b/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102896799998007, + "build_seconds": 9.582570386000015, + "workflow_seconds": null, + "total_seconds": 9.683599353999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.004706392000002779, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 9.683599353999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173909", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] +} diff --git a/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-warm.json b/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..6d888a74b --- /dev/null +++ b/results/gogs-moon/registry-screening-02/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10111357099998486, + "build_seconds": 1.5931755330000215, + "workflow_seconds": null, + "total_seconds": 1.6942891040000063, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.003638071000011678, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 1.6942891040000063 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173921", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] +} diff --git a/results/gogs-moon/registry-screening-02/runs/1-boringcache-cold.json b/results/gogs-moon/registry-screening-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..509ce0ceb --- /dev/null +++ b/results/gogs-moon/registry-screening-02/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20126020800000788, + "build_seconds": 7.471596422999994, + "workflow_seconds": null, + "total_seconds": 7.672856631000002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 3.0447996050000086, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 7.672856631000002 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173910", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] +} diff --git a/results/gogs-moon/registry-screening-02/runs/1-boringcache-warm.json b/results/gogs-moon/registry-screening-02/runs/1-boringcache-warm.json new file mode 100644 index 000000000..6d0bd4335 --- /dev/null +++ b/results/gogs-moon/registry-screening-02/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "gogs-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "gogs-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5a08ae847e4a3d1af5973f60b6312a4ee9a330d304a6aa91ab51aa3929423697", + "source": { + "repository": "gogs/gogs", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + } + ] + }, + "origin": { + "repository": "gogs/gogs", + "branch": "main", + "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the Gogs web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "gogs-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20114219899998886, + "build_seconds": 3.7727488040000026, + "workflow_seconds": null, + "total_seconds": 3.9738910029999914, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3115027010000091, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 3.9738910029999914 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "gogs-moon-series-registry-screening-02-s1-r37301997879-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "gogs/gogs", + "sha": "dbbd717e923694c36f41b0360515b4374bbe6ee0" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37301997879", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173920", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37301997879-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37301997879" + ] +} diff --git a/results/grpc/output-download-01/completions/37296157157.json b/results/grpc/output-download-01/completions/37296157157.json new file mode 100644 index 000000000..6bf4a4515 --- /dev/null +++ b/results/grpc/output-download-01/completions/37296157157.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "grpc", + "series_id": "output-download-01", + "sample": 1, + "run_id": "37296157157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296157157", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7192d33d863f1e7abb3f4ad9abe8eeb691310cd828a5f6ac2d107c0429bd110e" +} diff --git a/results/grpc/output-download-01/dispatches/1-grpc-grpc-bazel-fresh-benchmark.yml.json b/results/grpc/output-download-01/dispatches/1-grpc-grpc-bazel-fresh-benchmark.yml.json new file mode 100644 index 000000000..de52becc9 --- /dev/null +++ b/results/grpc/output-download-01/dispatches/1-grpc-grpc-bazel-fresh-benchmark.yml.json @@ -0,0 +1,33 @@ +{ + "case_id": "grpc", + "repository": "boringcache/benchmarks", + "workflow": "grpc-grpc-bazel-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "lane": "fresh", + "inputs": { + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "output-download-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T10:22:20Z", + "ref": "run-tool-family-cases", + "state": "requested", + "run_id": 37296157157, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296157157" +} diff --git a/results/grpc/output-download-01/report.json b/results/grpc/output-download-01/report.json new file mode 100644 index 000000000..a0bb74144 --- /dev/null +++ b/results/grpc/output-download-01/report.json @@ -0,0 +1,1087 @@ +{ + "schema_version": 1, + "plan_sha256": "a363a40e474799553b5fbaf158c0a060ab6b2bcee8aa9f9ab6bfc7e216553f50", + "complete": true, + "valid_for_comparison": true, + "failures": [], + "methodology_issues": [], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "grpc", + "series_id": "output-download-01", + "sample": 1, + "run_id": "37296157157", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296157157", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "7192d33d863f1e7abb3f4ad9abe8eeb691310cd828a5f6ac2d107c0429bd110e" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 3193, + "min": 3193, + "max": 3193 + }, + "storage": { + "median": 163734007, + "min": 163734007, + "max": 163734007 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "actions-cache", + "count": 1, + "measurement": { + "median": 93, + "min": 93, + "max": 93 + }, + "storage": { + "median": 163734007, + "min": 163734007, + "max": 163734007 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 2107, + "min": 2107, + "max": 2107 + }, + "storage": { + "median": 647876056, + "min": 647876056, + "max": 647876056 + }, + "storage_measured_count": 1 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 84, + "min": 84, + "max": 84 + }, + "storage": { + "median": 718160974, + "min": 718160974, + "max": 718160974 + }, + "storage_measured_count": 1 + }, + { + "phase": "cold", + "provider": "buildbuddy", + "count": 1, + "measurement": { + "median": 2092, + "min": 2092, + "max": 2092 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "buildbuddy", + "count": 1, + "measurement": { + "median": 122, + "min": 122, + "max": 122 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 3192, + "workflow_seconds": 3202, + "total_seconds": 3193, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 3193 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 163734007, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "complete": true, + "measured_bytes": 163734007, + "total_bytes": 163734007, + "observations": [ + { + "id": 8514390348, + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "ref": "refs/heads/run-tool-family-cases", + "version": "9e07caa64acd418e65f8cff95beceac634c3dcfc744079438f7ff22588f9d7ea", + "created_at": "2026-10-05T11:16:09.482022Z", + "last_accessed_at": "2026-10-05T11:16:09.482022Z", + "size_in_bytes": 163734007 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173723", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + }, + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 90, + "workflow_seconds": 97, + "total_seconds": 93, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 93 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 163734007, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "complete": true, + "measured_bytes": 163734007, + "total_bytes": 163734007, + "observations": [ + { + "id": 8514390348, + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "ref": "refs/heads/run-tool-family-cases", + "version": "9e07caa64acd418e65f8cff95beceac634c3dcfc744079438f7ff22588f9d7ea", + "created_at": "2026-10-05T11:16:09.482022Z", + "last_accessed_at": "2026-10-05T11:16:42.818492Z", + "size_in_bytes": 163734007 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173894", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + }, + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 2105, + "workflow_seconds": 2110, + "total_seconds": 2107, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 2107 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 647876056, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "complete": true, + "measured_bytes": 647876056, + "unmeasured_tags": [], + "total_bytes": 647876056, + "observations": [ + { + "requested_tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 647876056, + "compressed_size": 647876056, + "size": 647876056 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "bazel", + "resolved_tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173724", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + }, + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 72, + "workflow_seconds": 91, + "total_seconds": 84, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 84 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 718160974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "complete": true, + "measured_bytes": 718160974, + "unmeasured_tags": [], + "total_bytes": 718160974, + "observations": [ + { + "requested_tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 718160974, + "compressed_size": 718160974, + "size": 718160974 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "bazel", + "resolved_tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173896", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + }, + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "buildbuddy", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 2092, + "workflow_seconds": 2096, + "total_seconds": 2092, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 2092 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173722", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + }, + { + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "buildbuddy", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 122, + "workflow_seconds": 129, + "total_seconds": 122, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 122 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173895", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] + } + ], + "exclusions": [] +} diff --git a/results/grpc/output-download-01/report.md b/results/grpc/output-download-01/report.md new file mode 100644 index 000000000..9c863076c --- /dev/null +++ b/results/grpc/output-download-01/report.md @@ -0,0 +1,33 @@ +# grpc: output-download-01 + +Question: How do build and cache reuse time and measured storage compare for the same pinned workload? + +Measured scope: Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately + +Observations: 6/6 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | Actions Cache | 1 | 3193 | 3193–3193 | 163734007 | 1 | +| Warm build | Actions Cache | 1 | 93 | 93–93 | 163734007 | 1 | +| Cold build | BoringCache | 1 | 2107 | 2107–2107 | 647876056 | 1 | +| Warm build | BoringCache | 1 | 84 | 84–84 | 718160974 | 1 | +| Cold build | BuildBuddy | 1 | 2092 | 2092–2092 | unmeasured | 0 | +| Warm build | BuildBuddy | 1 | 122 | 122–122 | unmeasured | 0 | + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | Actions Cache | Cold build | 1 | 3192 | 3193 | 163734007 | github-actions-cache-api | not reported | [JSON](runs/1-actions-cache-cold.json) | +| 1 | Actions Cache | Warm build | 3 | 90 | 93 | 163734007 | github-actions-cache-api | hit | [JSON](runs/1-actions-cache-warm.json) | +| 1 | BoringCache | Cold build | 2 | 2105 | 2107 | 647876056 | boringcache-check | not reported | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 12 | 72 | 84 | 718160974 | boringcache-check | not reported | [JSON](runs/1-boringcache-warm.json) | +| 1 | BuildBuddy | Cold build | 0 | 2092 | 2092 | unmeasured | unmeasured | not reported | [JSON](runs/1-buildbuddy-cold.json) | +| 1 | BuildBuddy | Warm build | 0 | 122 | 122 | unmeasured | unmeasured | not reported | [JSON](runs/1-buildbuddy-warm.json) | + +[Full records and checks](report.json) diff --git a/results/grpc/output-download-01/runs/1-actions-cache-cold.json b/results/grpc/output-download-01/runs/1-actions-cache-cold.json new file mode 100644 index 000000000..69e5e7a64 --- /dev/null +++ b/results/grpc/output-download-01/runs/1-actions-cache-cold.json @@ -0,0 +1,157 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 1, + "build_seconds": 3192, + "workflow_seconds": 3202, + "total_seconds": 3193, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 3193 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 163734007, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "complete": true, + "measured_bytes": 163734007, + "total_bytes": 163734007, + "observations": [ + { + "id": 8514390348, + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "ref": "refs/heads/run-tool-family-cases", + "version": "9e07caa64acd418e65f8cff95beceac634c3dcfc744079438f7ff22588f9d7ea", + "created_at": "2026-10-05T11:16:09.482022Z", + "last_accessed_at": "2026-10-05T11:16:09.482022Z", + "size_in_bytes": 163734007 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173723", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/runs/1-actions-cache-warm.json b/results/grpc/output-download-01/runs/1-actions-cache-warm.json new file mode 100644 index 000000000..7a481ef3d --- /dev/null +++ b/results/grpc/output-download-01/runs/1-actions-cache-warm.json @@ -0,0 +1,157 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "actions-cache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 3, + "build_seconds": 90, + "workflow_seconds": 97, + "total_seconds": 93, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 93 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": 163734007, + "storage_source": "github-actions-cache-api", + "storage_breakdown": { + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "complete": true, + "measured_bytes": 163734007, + "total_bytes": 163734007, + "observations": [ + { + "id": 8514390348, + "key": "ac-grpc-bazel-series-output-download-01-s1-r37296157157-a1-cold", + "ref": "refs/heads/run-tool-family-cases", + "version": "9e07caa64acd418e65f8cff95beceac634c3dcfc744079438f7ff22588f9d7ea", + "created_at": "2026-10-05T11:16:09.482022Z", + "last_accessed_at": "2026-10-05T11:16:42.818492Z", + "size_in_bytes": 163734007 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173894", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/runs/1-boringcache-cold.json b/results/grpc/output-download-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..ad33274b9 --- /dev/null +++ b/results/grpc/output-download-01/runs/1-boringcache-cold.json @@ -0,0 +1,184 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2, + "build_seconds": 2105, + "workflow_seconds": 2110, + "total_seconds": 2107, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 2107 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 647876056, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "complete": true, + "measured_bytes": 647876056, + "unmeasured_tags": [], + "total_bytes": 647876056, + "observations": [ + { + "requested_tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 647876056, + "compressed_size": 647876056, + "size": 647876056 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "bazel", + "resolved_tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173724", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/runs/1-boringcache-warm.json b/results/grpc/output-download-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..2f6da1732 --- /dev/null +++ b/results/grpc/output-download-01/runs/1-boringcache-warm.json @@ -0,0 +1,184 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 12, + "build_seconds": 72, + "workflow_seconds": 91, + "total_seconds": 84, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 84 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": 718160974, + "storage_source": "boringcache-check", + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "complete": true, + "measured_bytes": 718160974, + "unmeasured_tags": [], + "total_bytes": 718160974, + "observations": [ + { + "requested_tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "tag": "grpc-bazel-series-output-download-01-s1-r37296157157-a1", + "status": "hit", + "cache_type": "kv", + "kv_total_size": 718160974, + "compressed_size": 718160974, + "size": 718160974 + } + ] + } + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "bazel", + "resolved_tags": [ + "grpc-bazel-series-output-download-01-s1-r37296157157-a1" + ], + "trust_state": { + "status": "restore_only", + "event_name": "workflow_dispatch", + "requested_policy": "restore", + "resolved_policy": "restore", + "write_allowed": false, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "trust-policy is restore.", + "next_step": "Use trust-policy stage or publish only when this job is trusted for that operation.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173896", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/runs/1-buildbuddy-cold.json b/results/grpc/output-download-01/runs/1-buildbuddy-cold.json new file mode 100644 index 000000000..33965e904 --- /dev/null +++ b/results/grpc/output-download-01/runs/1-buildbuddy-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "buildbuddy", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 2092, + "workflow_seconds": 2096, + "total_seconds": 2092, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 2092 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "cold", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173722", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/runs/1-buildbuddy-warm.json b/results/grpc/output-download-01/runs/1-buildbuddy-warm.json new file mode 100644 index 000000000..b553f73f9 --- /dev/null +++ b/results/grpc/output-download-01/runs/1-buildbuddy-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "grpc-bazel", + "strategy": "buildbuddy", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "bazel", + "adapter": "bazel", + "case": { + "schema_version": 1, + "case_id": "grpc", + "workspace": "boringcache/benchmarks", + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "origin": { + "repository": "boringcache/benchmark-grpc", + "branch": "main", + "revision": "00119b2080dd8c4d20cfb4647316a28417b4e1a6" + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/grpc-grpc-bazel-benchmark.yml", + "lane": "rolling", + "inputs": {} + }, + { + "path": ".github/workflows/grpc-grpc-bazel-fresh-benchmark.yml", + "lane": "fresh", + "inputs": {} + } + ], + "sync": "upstream-head", + "sync_minute": 7, + "cache_tags": { + "grpc-bazel-local": "{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 0, + "build_seconds": 122, + "workflow_seconds": 129, + "total_seconds": 122, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "build_and_reuse_seconds": 122 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "grpc/grpc", + "sha": "e1245717f658d10fee8fa0ea7b6f301a872330f2" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296157157", + "run_attempt": "1", + "job": "warm", + "workflow": "grpc: gRPC Bazel Fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173895", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296157157-1", + "series": { + "id": "output-download-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-latest" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296157157" + ] +} diff --git a/results/grpc/output-download-01/series.json b/results/grpc/output-download-01/series.json new file mode 100644 index 000000000..10a860877 --- /dev/null +++ b/results/grpc/output-download-01/series.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "case_id": "grpc", + "series_id": "output-download-01", + "created_at": "2026-10-05T10:22:17Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "17e1549cd1981da5cba45640fe509a9ef387e65638ff78a7730407889e4234e2", + "variant": null, + "workflow_inputs": { + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "How do build and cache reuse time and measured storage compare for the same pinned workload?", + "providers": [ + "boringcache", + "actions-cache", + "buildbuddy" + ], + "timed_scope": "Timed build entrypoint plus measured cache restore/setup; upstream dependency installation remains included when the entrypoint performs it; unmeasured post-job save is reported separately", + "primary_metric": "build_and_reuse_seconds", + "storage": "provider-reported", + "cache_scope": "Case-specific tags; fresh series use run and attempt identity; rolling series retain their declared cohort", + "sample_count": 10 + }, + "source": { + "repository": "grpc/grpc", + "pins": [ + { + "path": "upstream", + "revision": "e1245717f658d10fee8fa0ea7b6f301a872330f2", + "kind": "gitlink" + } + ] + }, + "verification": [ + "Pinned upstream recipe and build success", + "Nonempty executable csm_greeter_client and csm_greeter_server outputs, checked outside build timing" + ], + "plan_sha256": "a363a40e474799553b5fbaf158c0a060ab6b2bcee8aa9f9ab6bfc7e216553f50" +} diff --git a/results/helix-nix/canary-qualification-01/dispatches/1-nix-fresh-benchmark.yml.json b/results/helix-nix/canary-qualification-01/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..c3a989a84 --- /dev/null +++ b/results/helix-nix/canary-qualification-01/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "canary-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:34:11Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37303778364, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303778364" +} diff --git a/results/helix-nix/canary-qualification-01/dispatches/2-nix-fresh-benchmark.yml.json b/results/helix-nix/canary-qualification-01/dispatches/2-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..2c42a6ed9 --- /dev/null +++ b/results/helix-nix/canary-qualification-01/dispatches/2-nix-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "canary-qualification-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:34:13Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37303782303, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303782303" +} diff --git a/results/msgpack-sbt/registry-qualification-01/completions/37302718511.json b/results/msgpack-sbt/registry-qualification-01/completions/37302718511.json new file mode 100644 index 000000000..9f7a2fc6b --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/completions/37302718511.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302718511", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302718511", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f573580e4f886e0a21343f3377440101fa3dee54750ec852e996e3e5c926746c" +} diff --git a/results/msgpack-sbt/registry-qualification-01/completions/37302721578.json b/results/msgpack-sbt/registry-qualification-01/completions/37302721578.json new file mode 100644 index 000000000..7d193d384 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/completions/37302721578.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302721578", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302721578", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "23057a79736019f974fe24bd5d9448b6e0a2539f778f5f2e0ed1818c5806d449" +} diff --git a/results/msgpack-sbt/registry-qualification-01/report.json b/results/msgpack-sbt/registry-qualification-01/report.json new file mode 100644 index 000000000..a47f07d44 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/report.json @@ -0,0 +1,1210 @@ +{ + "schema_version": 1, + "plan_sha256": "796afbd8d1867db7442db953d38cb7e35d757fb18ef0d8825213983de643110e", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302718511", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302718511", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "f573580e4f886e0a21343f3377440101fa3dee54750ec852e996e3e5c926746c" + }, + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302721578", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302721578", + "source_sha": "09efe2188e21ac30bde55b760b6d3f0b23d24ac5", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "23057a79736019f974fe24bd5d9448b6e0a2539f778f5f2e0ed1818c5806d449" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110575599992444, + "build_seconds": 25.996618488999957, + "workflow_seconds": null, + "total_seconds": 26.09772424499988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014465399999608053, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 26.09772424499988 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174011", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1006594659999962, + "build_seconds": 14.459391225999994, + "workflow_seconds": null, + "total_seconds": 14.56005069199999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0011439679999938335, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.56005069199999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174032", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094350800000029, + "build_seconds": 38.32947641700002, + "workflow_seconds": null, + "total_seconds": 38.430419925000024, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.44028564100000267, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 38.430419925000024 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174012", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2015287419999936, + "build_seconds": 24.885599751, + "workflow_seconds": null, + "total_seconds": 25.087128492999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36464391500000204, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 25.087128492999994 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174031", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107761800000503, + "build_seconds": 39.63239714100001, + "workflow_seconds": null, + "total_seconds": 39.73347475900002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014527179999959117, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 39.73347475900002 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174015", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10077569999998559, + "build_seconds": 23.176340411000012, + "workflow_seconds": null, + "total_seconds": 23.277116110999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013771650000080626, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.277116110999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174036", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2013913069999944, + "build_seconds": 38.50265927199999, + "workflow_seconds": null, + "total_seconds": 38.70405057899998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9980765419999784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 38.70405057899998 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174014", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009555320000004, + "build_seconds": 23.221232052000005, + "workflow_seconds": null, + "total_seconds": 23.322187584000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.21567166099998758, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.322187584000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174035", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/registry-qualification-01/report.md b/results/msgpack-sbt/registry-qualification-01/report.md new file mode 100644 index 000000000..6e02aa698 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/report.md @@ -0,0 +1,37 @@ +# msgpack-sbt: registry-qualification-01 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 2 | unmeasured | 0 | +| Warm build | bazel-remote | 2 | unmeasured | 0 | +| Cold build | BoringCache | 2 | unmeasured | 0 | +| Warm build | BoringCache | 2 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10110575599992444 | 25.996618488999957 | 26.09772424499988 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.1006594659999962 | 14.459391225999994 | 14.56005069199999 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.10094350800000029 | 38.32947641700002 | 38.430419925000024 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.2015287419999936 | 24.885599751 | 25.087128492999994 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | +| 2 | bazel-remote | Cold build | 0.10107761800000503 | 39.63239714100001 | 39.73347475900002 | unmeasured | unmeasured | miss | [JSON](runs/2-bazel-remote-cold.json) | +| 2 | bazel-remote | Warm build | 0.10077569999998559 | 23.176340411000012 | 23.277116110999998 | unmeasured | unmeasured | hit | [JSON](runs/2-bazel-remote-warm.json) | +| 2 | BoringCache | Cold build | 0.2013913069999944 | 38.50265927199999 | 38.70405057899998 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 0.1009555320000004 | 23.221232052000005 | 23.322187584000005 | unmeasured | unmeasured | hit | [JSON](runs/2-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-cold.json b/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..d8727f17f --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-cold.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110575599992444, + "build_seconds": 25.996618488999957, + "workflow_seconds": null, + "total_seconds": 26.09772424499988, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014465399999608053, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 26.09772424499988 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174011", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-warm.json b/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..b6a19658d --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/1-bazel-remote-warm.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1006594659999962, + "build_seconds": 14.459391225999994, + "workflow_seconds": null, + "total_seconds": 14.56005069199999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0011439679999938335, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.56005069199999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174032", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-cold.json b/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..53949394d --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10094350800000029, + "build_seconds": 38.32947641700002, + "workflow_seconds": null, + "total_seconds": 38.430419925000024, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.44028564100000267, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 38.430419925000024 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174012", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-warm.json b/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..9a6de6084 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2015287419999936, + "build_seconds": 24.885599751, + "workflow_seconds": null, + "total_seconds": 25.087128492999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.36464391500000204, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 25.087128492999994 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s1-r37302718511-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302718511", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174031", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302718511-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302718511" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-cold.json b/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-cold.json new file mode 100644 index 000000000..7a3897758 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-cold.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10107761800000503, + "build_seconds": 39.63239714100001, + "workflow_seconds": null, + "total_seconds": 39.73347475900002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014527179999959117, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 39.73347475900002 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174015", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-warm.json b/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-warm.json new file mode 100644 index 000000000..ac1f5429f --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/2-bazel-remote-warm.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10077569999998559, + "build_seconds": 23.176340411000012, + "workflow_seconds": null, + "total_seconds": 23.277116110999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013771650000080626, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.277116110999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174036", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-cold.json b/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..1542aec48 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-cold.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2013913069999944, + "build_seconds": 38.50265927199999, + "workflow_seconds": null, + "total_seconds": 38.70405057899998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.9980765419999784, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 38.70405057899998 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174014", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] +} diff --git a/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-warm.json b/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..884d4ba49 --- /dev/null +++ b/results/msgpack-sbt/registry-qualification-01/runs/2-boringcache-warm.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009555320000004, + "build_seconds": 23.221232052000005, + "workflow_seconds": null, + "total_seconds": 23.322187584000005, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.21567166099998758, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.322187584000005 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-qualification-01-s2-r37302721578-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302721578", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174035", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302721578-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302721578" + ] +} diff --git a/results/msgpack-sbt/registry-screening-03/completions/37302010045.json b/results/msgpack-sbt/registry-screening-03/completions/37302010045.json new file mode 100644 index 000000000..9ca64d841 --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/completions/37302010045.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302010045", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302010045", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "890ffdc934b57d911ca4cf05267c58040d3f0449a5144c011fbdcfe5c08f06e9" +} diff --git a/results/msgpack-sbt/registry-screening-03/report.json b/results/msgpack-sbt/registry-screening-03/report.json new file mode 100644 index 000000000..323b5ad3f --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/report.json @@ -0,0 +1,633 @@ +{ + "schema_version": 1, + "plan_sha256": "d599b25c1d0c358d407f5915f5c3a7648565b1d4e16ffd5ece73fb88cac437b4", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "msgpack-sbt", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302010045", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302010045", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "890ffdc934b57d911ca4cf05267c58040d3f0449a5144c011fbdcfe5c08f06e9" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10114316599999995, + "build_seconds": 30.065101705000004, + "workflow_seconds": null, + "total_seconds": 30.166244871000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013599889999937886, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 30.166244871000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173917", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102147899999636, + "build_seconds": 23.825571293000024, + "workflow_seconds": null, + "total_seconds": 23.92659277200002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014791060000050038, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.92659277200002 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173928", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10092451200000596, + "build_seconds": 39.360577026, + "workflow_seconds": null, + "total_seconds": 39.46150153800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.489295205000019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 39.46150153800001 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173916", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] + }, + { + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101283900000624, + "build_seconds": 21.556729329999996, + "workflow_seconds": null, + "total_seconds": 21.657742169000002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.24145116199998995, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 21.657742169000002 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173929", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] + } + ], + "exclusions": [] +} diff --git a/results/msgpack-sbt/registry-screening-03/report.md b/results/msgpack-sbt/registry-screening-03/report.md new file mode 100644 index 000000000..16e7079ac --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/report.md @@ -0,0 +1,33 @@ +# msgpack-sbt: registry-screening-03 + +Question: Does sbt reuse MessagePack compilation and test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 4/4 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 1 | unmeasured | 0 | +| Warm build | bazel-remote | 1 | unmeasured | 0 | +| Cold build | BoringCache | 1 | unmeasured | 0 | +| Warm build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10114316599999995 | 30.065101705000004 | 30.166244871000004 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10102147899999636 | 23.825571293000024 | 23.92659277200002 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.10092451200000596 | 39.360577026 | 39.46150153800001 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.10101283900000624 | 21.556729329999996 | 21.657742169000002 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-cold.json b/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..120e3d069 --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-cold.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10114316599999995, + "build_seconds": 30.065101705000004, + "workflow_seconds": null, + "total_seconds": 30.166244871000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0013599889999937886, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 30.166244871000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173917", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] +} diff --git a/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-warm.json b/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..6fb4ff592 --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/runs/1-bazel-remote-warm.json @@ -0,0 +1,140 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102147899999636, + "build_seconds": 23.825571293000024, + "workflow_seconds": null, + "total_seconds": 23.92659277200002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014791060000050038, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 23.92659277200002 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173928", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] +} diff --git a/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-cold.json b/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-cold.json new file mode 100644 index 000000000..6a4321f9f --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-cold.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10092451200000596, + "build_seconds": 39.360577026, + "workflow_seconds": null, + "total_seconds": 39.46150153800001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.489295205000019, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 39.46150153800001 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173916", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] +} diff --git a/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-warm.json b/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-warm.json new file mode 100644 index 000000000..b7a44bf45 --- /dev/null +++ b/results/msgpack-sbt/registry-screening-03/runs/1-boringcache-warm.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "msgpack-sbt", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "msgpack-sbt", + "workspace": "boringcache/benchmarks", + "definition_sha256": "e7eefb10f88ec572f38a8bff543d2b1abac7eb1fb2cf99a7ef278e4105a7650a", + "source": { + "repository": "msgpack/msgpack-java", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "pins": [ + { + "path": "proposal.md", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4", + "kind": "snapshot" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "msgpack/msgpack-java", + "branch": "main", + "revision": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does sbt reuse MessagePack compilation and test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "primary_metric": "correctness", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "msgpack-sbt" + } + } + ], + "sync": "fixed", + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10101283900000624, + "build_seconds": 21.556729329999996, + "workflow_seconds": null, + "total_seconds": 21.657742169000002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.24145116199998995, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 21.657742169000002 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "msgpack-sbt-series-registry-screening-03-s1-r37302010045-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "msgpack/msgpack-java", + "sha": "5c28fbad6cd360d8ea822df109830ae2517a7ae4" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302010045", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173929", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302010045-1", + "series": { + "id": "registry-screening-03", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302010045" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/completions/37302929873.json b/results/opencut-moon/registry-qualification-01/completions/37302929873.json new file mode 100644 index 000000000..381f85aa6 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/completions/37302929873.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302929873", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302929873", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c0e9cb3efb9b12c5aa2c25f24461e255ca60f47a2580ff454898e4e31b866b23" +} diff --git a/results/opencut-moon/registry-qualification-01/completions/37302932865.json b/results/opencut-moon/registry-qualification-01/completions/37302932865.json new file mode 100644 index 000000000..720090568 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/completions/37302932865.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302932865", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302932865", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "55fb7c3f486e1b081674b0bbcbadea8161bcd8329dc144c2946e65686b07f82f" +} diff --git a/results/opencut-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json b/results/opencut-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..64875682b --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:26:19Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302929873, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302929873" +} diff --git a/results/opencut-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json b/results/opencut-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..67e2fb9c1 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/dispatches/2-reapi-fresh-benchmark.yml.json @@ -0,0 +1,35 @@ +{ + "case_id": "opencut-moon", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-qualification-01", + "sample": "2" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:26:21Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37302932865, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302932865" +} diff --git a/results/opencut-moon/registry-qualification-01/report.json b/results/opencut-moon/registry-qualification-01/report.json new file mode 100644 index 000000000..2ce376402 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/report.json @@ -0,0 +1,1202 @@ +{ + "schema_version": 1, + "plan_sha256": "726ca855e2aad904fe6a34e52fb84e252e35aa6504556537fa0f7c979f9c0bc7", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37302929873", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302929873", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "c0e9cb3efb9b12c5aa2c25f24461e255ca60f47a2580ff454898e4e31b866b23" + }, + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37302932865", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302932865", + "source_sha": "c848083da1f4febbae09a6ac66257da43810accc", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "55fb7c3f486e1b081674b0bbcbadea8161bcd8329dc144c2946e65686b07f82f" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110312699998758, + "build_seconds": 14.90259992099999, + "workflow_seconds": null, + "total_seconds": 15.003703047999977, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014272399999981644, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.003703047999977 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174039", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10098400499999372, + "build_seconds": 11.398400035999998, + "workflow_seconds": null, + "total_seconds": 11.499384040999992, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016290470000086543, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.499384040999992 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174041", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2010586919999966, + "build_seconds": 10.875096673999998, + "workflow_seconds": null, + "total_seconds": 11.076155365999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6899565410000008, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.076155365999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174040", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103631699999482, + "build_seconds": 11.797382260999996, + "workflow_seconds": null, + "total_seconds": 11.89841857799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.16414089200000603, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.89841857799999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174042", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010827279999944, + "build_seconds": 15.102131216000032, + "workflow_seconds": null, + "total_seconds": 15.203213944000026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001432246000035775, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.203213944000026 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174037", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110542299999992, + "build_seconds": 10.692360459, + "workflow_seconds": null, + "total_seconds": 10.793465882, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0025870329999975183, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 10.793465882 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174043", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128874499999938, + "build_seconds": 14.586881934000004, + "workflow_seconds": null, + "total_seconds": 14.788170679000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.0371327299999962, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.788170679000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174038", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085053600001004, + "build_seconds": 11.697542224000003, + "workflow_seconds": null, + "total_seconds": 11.798392760000013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.20673704200000032, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.798392760000013 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174044", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/registry-qualification-01/report.md b/results/opencut-moon/registry-qualification-01/report.md new file mode 100644 index 000000000..d6878d681 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/report.md @@ -0,0 +1,37 @@ +# opencut-moon: registry-qualification-01 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 2 | unmeasured | 0 | +| Warm build | bazel-remote | 2 | unmeasured | 0 | +| Cold build | BoringCache | 2 | unmeasured | 0 | +| Warm build | BoringCache | 2 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10110312699998758 | 14.90259992099999 | 15.003703047999977 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10098400499999372 | 11.398400035999998 | 11.499384040999992 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.2010586919999966 | 10.875096673999998 | 11.076155365999995 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.10103631699999482 | 11.797382260999996 | 11.89841857799999 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | +| 2 | bazel-remote | Cold build | 0.1010827279999944 | 15.102131216000032 | 15.203213944000026 | unmeasured | unmeasured | miss | [JSON](runs/2-bazel-remote-cold.json) | +| 2 | bazel-remote | Warm build | 0.10110542299999992 | 10.692360459 | 10.793465882 | unmeasured | unmeasured | hit | [JSON](runs/2-bazel-remote-warm.json) | +| 2 | BoringCache | Cold build | 0.20128874499999938 | 14.586881934000004 | 14.788170679000004 | unmeasured | unmeasured | miss | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 0.10085053600001004 | 11.697542224000003 | 11.798392760000013 | unmeasured | unmeasured | hit | [JSON](runs/2-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-cold.json b/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..68908c111 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110312699998758, + "build_seconds": 14.90259992099999, + "workflow_seconds": null, + "total_seconds": 15.003703047999977, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014272399999981644, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.003703047999977 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174039", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-warm.json b/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..d1846a1f8 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10098400499999372, + "build_seconds": 11.398400035999998, + "workflow_seconds": null, + "total_seconds": 11.499384040999992, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0016290470000086543, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.499384040999992 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174041", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/1-boringcache-cold.json b/results/opencut-moon/registry-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..380c53b0f --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2010586919999966, + "build_seconds": 10.875096673999998, + "workflow_seconds": null, + "total_seconds": 11.076155365999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.6899565410000008, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.076155365999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174040", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/1-boringcache-warm.json b/results/opencut-moon/registry-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..0f8a064d5 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103631699999482, + "build_seconds": 11.797382260999996, + "workflow_seconds": null, + "total_seconds": 11.89841857799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.16414089200000603, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.89841857799999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s1-r37302929873-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302929873", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174042", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302929873-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302929873" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-cold.json b/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-cold.json new file mode 100644 index 000000000..e2263137f --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010827279999944, + "build_seconds": 15.102131216000032, + "workflow_seconds": null, + "total_seconds": 15.203213944000026, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001432246000035775, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.203213944000026 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174037", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-warm.json b/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-warm.json new file mode 100644 index 000000000..4420a4b9b --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/2-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10110542299999992, + "build_seconds": 10.692360459, + "workflow_seconds": null, + "total_seconds": 10.793465882, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0025870329999975183, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 10.793465882 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174043", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/2-boringcache-cold.json b/results/opencut-moon/registry-qualification-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..dd1cc1d19 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/2-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20128874499999938, + "build_seconds": 14.586881934000004, + "workflow_seconds": null, + "total_seconds": 14.788170679000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.0371327299999962, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.788170679000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174038", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] +} diff --git a/results/opencut-moon/registry-qualification-01/runs/2-boringcache-warm.json b/results/opencut-moon/registry-qualification-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..6c16d9625 --- /dev/null +++ b/results/opencut-moon/registry-qualification-01/runs/2-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10085053600001004, + "build_seconds": 11.697542224000003, + "workflow_seconds": null, + "total_seconds": 11.798392760000013, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.20673704200000032, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.798392760000013 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-qualification-01-s2-r37302932865-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302932865", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174044", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302932865-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302932865" + ] +} diff --git a/results/opencut-moon/registry-screening-02/completions/37302001164.json b/results/opencut-moon/registry-screening-02/completions/37302001164.json new file mode 100644 index 000000000..cf7ac8b9f --- /dev/null +++ b/results/opencut-moon/registry-screening-02/completions/37302001164.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302001164", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache opencut-moon warm: failure" + ], + "evidence_manifest_sha256": "09e17dc66880bf03926e144770b393e02d183ee23c1675b0d1a3854ebdb6d5d6" +} diff --git a/results/opencut-moon/registry-screening-02/report.json b/results/opencut-moon/registry-screening-02/report.json new file mode 100644 index 000000000..c57771d6f --- /dev/null +++ b/results/opencut-moon/registry-screening-02/report.json @@ -0,0 +1,506 @@ +{ + "schema_version": 1, + "plan_sha256": "1cb918b67aa5053d594ead632e738134735a9a7d9b949a4d4291b2768ef0f1f8", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302001164", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache opencut-moon warm: failure" + ], + "evidence_manifest_sha256": "09e17dc66880bf03926e144770b393e02d183ee23c1675b0d1a3854ebdb6d5d6" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302001164", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302001164", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache opencut-moon warm: failure" + ], + "evidence_manifest_sha256": "09e17dc66880bf03926e144770b393e02d183ee23c1675b0d1a3854ebdb6d5d6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "warm", + "boringcache" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011117870000362, + "build_seconds": 15.10769691400003, + "workflow_seconds": null, + "total_seconds": 15.208808701000066, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001374484999928427, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.208808701000066 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173912", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103412799998068, + "build_seconds": 10.993322974999984, + "workflow_seconds": null, + "total_seconds": 11.094357102999965, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0021235250000017913, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.094357102999965 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173923", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1008831449999974, + "build_seconds": 14.597016275000009, + "workflow_seconds": null, + "total_seconds": 14.697899420000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.0165929129999967, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.697899420000006 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173913", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/registry-screening-02/report.md b/results/opencut-moon/registry-screening-02/report.md new file mode 100644 index 000000000..18f4a9712 --- /dev/null +++ b/results/opencut-moon/registry-screening-02/report.md @@ -0,0 +1,40 @@ +# opencut-moon: registry-screening-02 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 3/4 recorded; 0 failed; 1 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 1 | unmeasured | 0 | +| Warm build | bazel-remote | 1 | unmeasured | 0 | +| Cold build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37302001164 failed completion checks: + +- Workflow concluded failure +- boringcache opencut-moon warm: failure + +Missing observations: + +- Sample 1, BoringCache, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.1011117870000362 | 15.10769691400003 | 15.208808701000066 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10103412799998068 | 10.993322974999984 | 11.094357102999965 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.1008831449999974 | 14.597016275000009 | 14.697899420000006 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-cold.json b/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..39351467b --- /dev/null +++ b/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1011117870000362, + "build_seconds": 15.10769691400003, + "workflow_seconds": null, + "total_seconds": 15.208808701000066, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.001374484999928427, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 15.208808701000066 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173912", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] +} diff --git a/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-warm.json b/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..fb7adba10 --- /dev/null +++ b/results/opencut-moon/registry-screening-02/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10103412799998068, + "build_seconds": 10.993322974999984, + "workflow_seconds": null, + "total_seconds": 11.094357102999965, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0021235250000017913, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.094357102999965 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173923", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] +} diff --git a/results/opencut-moon/registry-screening-02/runs/1-boringcache-cold.json b/results/opencut-moon/registry-screening-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..68b0dd558 --- /dev/null +++ b/results/opencut-moon/registry-screening-02/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "eda0c45e12fb18b4ec347c6f7af2fa11fb1339cf771abb444e767d1952c82c63", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1008831449999974, + "build_seconds": 14.597016275000009, + "workflow_seconds": null, + "total_seconds": 14.697899420000006, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 1.0165929129999967, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.697899420000006 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-02-s1-r37302001164-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302001164", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "main", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173913", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302001164-1", + "series": { + "id": "registry-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302001164" + ] +} diff --git a/results/opencut-moon/registry-screening-04/completions/37302476504.json b/results/opencut-moon/registry-screening-04/completions/37302476504.json new file mode 100644 index 000000000..0a5aa13d5 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/completions/37302476504.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37302476504", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302476504", + "source_sha": "a15241e5d23058d6fa8890ab4e16fd15e319d417", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "958d86ab1a602a20ac8b3b656219b3b598ff8672fea1e5a947f6e520f77c796c" +} diff --git a/results/opencut-moon/registry-screening-04/report.json b/results/opencut-moon/registry-screening-04/report.json new file mode 100644 index 000000000..45dd8db90 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/report.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "plan_sha256": "7016af095356764099bf0da79cec88f91affb18267ec18b5ebebdc310acdff72", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "opencut-moon", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37302476504", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302476504", + "source_sha": "a15241e5d23058d6fa8890ab4e16fd15e319d417", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "958d86ab1a602a20ac8b3b656219b3b598ff8672fea1e5a947f6e520f77c796c" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108930599999866, + "build_seconds": 16.494646345999996, + "workflow_seconds": null, + "total_seconds": 16.595735651999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014613210000078425, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 16.595735651999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173975", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10100244499999889, + "build_seconds": 11.088584404000002, + "workflow_seconds": null, + "total_seconds": 11.189586849000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015361600000005637, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.189586849000001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173978", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014246630000045, + "build_seconds": 10.773953866, + "workflow_seconds": null, + "total_seconds": 10.975378529000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.8072748369999942, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 10.975378529000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173974", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] + }, + { + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10091571799999599, + "build_seconds": 8.672696470000005, + "workflow_seconds": null, + "total_seconds": 8.773612188000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.28139467700000864, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.773612188000001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173977", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] + } + ], + "exclusions": [] +} diff --git a/results/opencut-moon/registry-screening-04/report.md b/results/opencut-moon/registry-screening-04/report.md new file mode 100644 index 000000000..f5fc39649 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/report.md @@ -0,0 +1,33 @@ +# opencut-moon: registry-screening-04 + +Question: Does Moon restore the OpenCut web build on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 4/4 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 1 | unmeasured | 0 | +| Warm build | bazel-remote | 1 | unmeasured | 0 | +| Cold build | BoringCache | 1 | unmeasured | 0 | +| Warm build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10108930599999866 | 16.494646345999996 | 16.595735651999995 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10100244499999889 | 11.088584404000002 | 11.189586849000001 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.2014246630000045 | 10.773953866 | 10.975378529000004 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.10091571799999599 | 8.672696470000005 | 8.773612188000001 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-cold.json b/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..7314c07e8 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10108930599999866, + "build_seconds": 16.494646345999996, + "workflow_seconds": null, + "total_seconds": 16.595735651999995, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0014613210000078425, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 16.595735651999995 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173975", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] +} diff --git a/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-warm.json b/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..b33e51a92 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10100244499999889, + "build_seconds": 11.088584404000002, + "workflow_seconds": null, + "total_seconds": 11.189586849000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0015361600000005637, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.189586849000001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173978", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] +} diff --git a/results/opencut-moon/registry-screening-04/runs/1-boringcache-cold.json b/results/opencut-moon/registry-screening-04/runs/1-boringcache-cold.json new file mode 100644 index 000000000..fe2655b01 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014246630000045, + "build_seconds": 10.773953866, + "workflow_seconds": null, + "total_seconds": 10.975378529000004, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.8072748369999942, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 10.975378529000004 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173974", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] +} diff --git a/results/opencut-moon/registry-screening-04/runs/1-boringcache-warm.json b/results/opencut-moon/registry-screening-04/runs/1-boringcache-warm.json new file mode 100644 index 000000000..251b66d78 --- /dev/null +++ b/results/opencut-moon/registry-screening-04/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "opencut-moon", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "opencut-moon", + "workspace": "boringcache/benchmarks", + "definition_sha256": "419fe1bc2d29badcc93444ce7d479b1e0e4b1a4022f11a0ad7f862715e776b3e", + "source": { + "repository": "OpenCut-app/OpenCut", + "revision": "e668010778568641babef2cc40be4703ae6916d6", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + } + ] + }, + "origin": { + "repository": "OpenCut-app/OpenCut", + "branch": "main", + "revision": "e668010778568641babef2cc40be4703ae6916d6" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Moon restore the OpenCut web build on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "opencut-moon" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10091571799999599, + "build_seconds": 8.672696470000005, + "workflow_seconds": null, + "total_seconds": 8.773612188000001, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.28139467700000864, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.773612188000001 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "opencut-moon-series-registry-screening-04-s1-r37302476504-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "OpenCut-app/OpenCut", + "sha": "e668010778568641babef2cc40be4703ae6916d6" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302476504", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173977", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302476504-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302476504" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/completions/37303666267.json b/results/stackstorm-pants/registry-qualification-01/completions/37303666267.json new file mode 100644 index 000000000..4bceb8ff1 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/completions/37303666267.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37303666267", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303666267", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2793052a56437ecb28302e51716a9cd954392e5328c64304e6f1bc3171eb4d32" +} diff --git a/results/stackstorm-pants/registry-qualification-01/completions/37303670119.json b/results/stackstorm-pants/registry-qualification-01/completions/37303670119.json new file mode 100644 index 000000000..965c8f6d9 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/completions/37303670119.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37303670119", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303670119", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "314c3c10a84c6538172ddcd5977feda6592c73d39bc987efa4250cd1757f7808" +} diff --git a/results/stackstorm-pants/registry-qualification-01/report.json b/results/stackstorm-pants/registry-qualification-01/report.json new file mode 100644 index 000000000..8c3adc6c5 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/report.json @@ -0,0 +1,1202 @@ +{ + "schema_version": 1, + "plan_sha256": "ae3ee8ca9d1fcd7b05d186e12a52dd03b42eebaf92f9480c27ea9bd632ac8a58", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 1, + "run_id": "37303666267", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303666267", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "2793052a56437ecb28302e51716a9cd954392e5328c64304e6f1bc3171eb4d32" + }, + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-qualification-01", + "sample": 2, + "run_id": "37303670119", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303670119", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "314c3c10a84c6538172ddcd5977feda6592c73d39bc987efa4250cd1757f7808" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 2, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10090506299999902, + "build_seconds": 18.365842974999993, + "workflow_seconds": null, + "total_seconds": 18.46674803799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.002226952999990317, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 18.46674803799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174082", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122621899999729, + "build_seconds": 13.986136833999993, + "workflow_seconds": null, + "total_seconds": 14.08736305299999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0037473640000058595, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.08736305299999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174094", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20136627799999474, + "build_seconds": 25.576910379999987, + "workflow_seconds": null, + "total_seconds": 25.77827665799998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7042534900000135, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 25.77827665799998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174081", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135146399999826, + "build_seconds": 11.666110438999993, + "workflow_seconds": null, + "total_seconds": 11.867461902999992, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.38366340700000023, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.867461902999992 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174095", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10105914099999325, + "build_seconds": 28.80764801299999, + "workflow_seconds": null, + "total_seconds": 28.908707153999984, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0030842780000028824, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 28.908707153999984 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174083", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10072521000000734, + "build_seconds": 8.28355738900001, + "workflow_seconds": null, + "total_seconds": 8.384282599000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0025049190000032695, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.384282599000016 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174097", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014053409999974, + "build_seconds": 27.876364369, + "workflow_seconds": null, + "total_seconds": 28.07776971, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7934625729999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 28.07776971 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174084", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145238800000698, + "build_seconds": 13.613789486999991, + "workflow_seconds": null, + "total_seconds": 13.815241874999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3499625970000011, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 13.815241874999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174098", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/registry-qualification-01/report.md b/results/stackstorm-pants/registry-qualification-01/report.md new file mode 100644 index 000000000..07f652c8d --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/report.md @@ -0,0 +1,37 @@ +# stackstorm-pants: registry-qualification-01 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 8/8 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 2 | unmeasured | 0 | +| Warm build | bazel-remote | 2 | unmeasured | 0 | +| Cold build | BoringCache | 2 | unmeasured | 0 | +| Warm build | BoringCache | 2 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.10090506299999902 | 18.365842974999993 | 18.46674803799999 | unmeasured | unmeasured | not reported | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10122621899999729 | 13.986136833999993 | 14.08736305299999 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.20136627799999474 | 25.576910379999987 | 25.77827665799998 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.20135146399999826 | 11.666110438999993 | 11.867461902999992 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | +| 2 | bazel-remote | Cold build | 0.10105914099999325 | 28.80764801299999 | 28.908707153999984 | unmeasured | unmeasured | not reported | [JSON](runs/2-bazel-remote-cold.json) | +| 2 | bazel-remote | Warm build | 0.10072521000000734 | 8.28355738900001 | 8.384282599000016 | unmeasured | unmeasured | hit | [JSON](runs/2-bazel-remote-warm.json) | +| 2 | BoringCache | Cold build | 0.2014053409999974 | 27.876364369 | 28.07776971 | unmeasured | unmeasured | not reported | [JSON](runs/2-boringcache-cold.json) | +| 2 | BoringCache | Warm build | 0.20145238800000698 | 13.613789486999991 | 13.815241874999998 | unmeasured | unmeasured | hit | [JSON](runs/2-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-cold.json b/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..d20b63cc6 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10090506299999902, + "build_seconds": 18.365842974999993, + "workflow_seconds": null, + "total_seconds": 18.46674803799999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.002226952999990317, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 18.46674803799999 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174082", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-warm.json b/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..a3ad1728d --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10122621899999729, + "build_seconds": 13.986136833999993, + "workflow_seconds": null, + "total_seconds": 14.08736305299999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0037473640000058595, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 14.08736305299999 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174094", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-cold.json b/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-cold.json new file mode 100644 index 000000000..52b0b5146 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20136627799999474, + "build_seconds": 25.576910379999987, + "workflow_seconds": null, + "total_seconds": 25.77827665799998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7042534900000135, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 25.77827665799998 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174081", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-warm.json b/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-warm.json new file mode 100644 index 000000000..d59b91712 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135146399999826, + "build_seconds": 11.666110438999993, + "workflow_seconds": null, + "total_seconds": 11.867461902999992, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.38366340700000023, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.867461902999992 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s1-r37303666267-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303666267", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174095", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303666267-1", + "series": { + "id": "registry-qualification-01", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303666267" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-cold.json b/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-cold.json new file mode 100644 index 000000000..af5ce65f3 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10105914099999325, + "build_seconds": 28.80764801299999, + "workflow_seconds": null, + "total_seconds": 28.908707153999984, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0030842780000028824, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 28.908707153999984 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174083", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-warm.json b/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-warm.json new file mode 100644 index 000000000..994eed60c --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/2-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10072521000000734, + "build_seconds": 8.28355738900001, + "workflow_seconds": null, + "total_seconds": 8.384282599000016, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0025049190000032695, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 8.384282599000016 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174097", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-cold.json b/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-cold.json new file mode 100644 index 000000000..bf48ed6e6 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.2014053409999974, + "build_seconds": 27.876364369, + "workflow_seconds": null, + "total_seconds": 28.07776971, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.7934625729999993, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 28.07776971 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174084", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] +} diff --git a/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-warm.json b/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-warm.json new file mode 100644 index 000000000..4bd287785 --- /dev/null +++ b/results/stackstorm-pants/registry-qualification-01/runs/2-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "7a8e1e072b03460362695acc5c55c3eed06c89202bd5b62be83f816b3d314abe", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20145238800000698, + "build_seconds": 13.613789486999991, + "workflow_seconds": null, + "total_seconds": 13.815241874999998, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3499625970000011, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 13.815241874999998 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-qualification-01-s2-r37303670119-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303670119", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174098", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303670119-1", + "series": { + "id": "registry-qualification-01", + "sample": 2 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303670119" + ] +} diff --git a/results/stackstorm-pants/registry-screening-02/completions/37302004103.json b/results/stackstorm-pants/registry-screening-02/completions/37302004103.json new file mode 100644 index 000000000..7ad9ef4c8 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-02/completions/37302004103.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302004103", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "ede5e77e9b757f31622697cec3fa93f8667cb0fd0d6d2cf1e5de44c94b3a8b35" +} diff --git a/results/stackstorm-pants/registry-screening-02/report.json b/results/stackstorm-pants/registry-screening-02/report.json new file mode 100644 index 000000000..6529a03d1 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-02/report.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "plan_sha256": "9fd0981ad3a1f8c068ad8be0ca85a799d4eef6d67e94e2bbf5851e9cce9662e6", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302004103", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "ede5e77e9b757f31622697cec3fa93f8667cb0fd0d6d2cf1e5de44c94b3a8b35" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-02", + "sample": 1, + "run_id": "37302004103", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302004103", + "source_sha": "b32e311b56132e7527aceada2aaae9f957431c63", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "ede5e77e9b757f31622697cec3fa93f8667cb0fd0d6d2cf1e5de44c94b3a8b35" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/stackstorm-pants/registry-screening-02/report.md b/results/stackstorm-pants/registry-screening-02/report.md new file mode 100644 index 000000000..251480313 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-02/report.md @@ -0,0 +1,38 @@ +# stackstorm-pants: registry-screening-02 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 0/4 recorded; 0 failed; 4 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37302004103 failed completion checks: + +- Workflow concluded failure +- boringcache stackstorm-pants cold: failure +- bazel-remote stackstorm-pants cold: failure + +Missing observations: + +- Sample 1, BoringCache, Cold build +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/registry-screening-03/completions/37302345686.json b/results/stackstorm-pants/registry-screening-03/completions/37302345686.json new file mode 100644 index 000000000..5e90a7d9c --- /dev/null +++ b/results/stackstorm-pants/registry-screening-03/completions/37302345686.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302345686", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686", + "source_sha": "6aebc8f9f33a9e8de209e747b48c0a4f2fdda60a", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "84e6a78d443c929f68096abc6c210d3a6783cfdd3f5fd444bed40c34c7b957cf" +} diff --git a/results/stackstorm-pants/registry-screening-03/report.json b/results/stackstorm-pants/registry-screening-03/report.json new file mode 100644 index 000000000..898b017cb --- /dev/null +++ b/results/stackstorm-pants/registry-screening-03/report.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "plan_sha256": "7cd6568e943d76372434e5ba184fd8204575671dc220d2c042b3e6cbd9200034", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302345686", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686", + "source_sha": "6aebc8f9f33a9e8de209e747b48c0a4f2fdda60a", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "84e6a78d443c929f68096abc6c210d3a6783cfdd3f5fd444bed40c34c7b957cf" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-03", + "sample": 1, + "run_id": "37302345686", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302345686", + "source_sha": "6aebc8f9f33a9e8de209e747b48c0a4f2fdda60a", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "boringcache stackstorm-pants cold: failure", + "bazel-remote stackstorm-pants cold: failure" + ], + "evidence_manifest_sha256": "84e6a78d443c929f68096abc6c210d3a6783cfdd3f5fd444bed40c34c7b957cf" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/stackstorm-pants/registry-screening-03/report.md b/results/stackstorm-pants/registry-screening-03/report.md new file mode 100644 index 000000000..b2136ed34 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-03/report.md @@ -0,0 +1,38 @@ +# stackstorm-pants: registry-screening-03 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 0/4 recorded; 0 failed; 4 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37302345686 failed completion checks: + +- Workflow concluded failure +- boringcache stackstorm-pants cold: failure +- bazel-remote stackstorm-pants cold: failure + +Missing observations: + +- Sample 1, BoringCache, Cold build +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/registry-screening-04/completions/37302896058.json b/results/stackstorm-pants/registry-screening-04/completions/37302896058.json new file mode 100644 index 000000000..450ced094 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/completions/37302896058.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37302896058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302896058", + "source_sha": "d63d3593c150f7471c7419a5bac6827c2979c3ef", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d9b28308dd0b7ffbc7d745fd577148d8ec4449eacad97cdfa729ecb399e905ed" +} diff --git a/results/stackstorm-pants/registry-screening-04/report.json b/results/stackstorm-pants/registry-screening-04/report.json new file mode 100644 index 000000000..6e9f9673e --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/report.json @@ -0,0 +1,629 @@ +{ + "schema_version": 1, + "plan_sha256": "e085a1693be2da021182976715305294e9d52fdc467d0e5f4b3b174b1a78d1d7", + "complete": true, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": true, + "missing_completions": [], + "failed_completions": [], + "completions": [ + { + "schema_version": 1, + "case_id": "stackstorm-pants", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37302896058", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37302896058", + "source_sha": "d63d3593c150f7471c7419a5bac6827c2979c3ef", + "github_conclusion": "success", + "status": "success", + "errors": [], + "evidence_manifest_sha256": "d9b28308dd0b7ffbc7d745fd577148d8ec4449eacad97cdfa729ecb399e905ed" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [], + "summaries": [ + { + "phase": "cold", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "bazel-remote", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "warm", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010307130000001, + "build_seconds": 27.520753834999994, + "workflow_seconds": null, + "total_seconds": 27.621784547999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0030729709999945953, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 27.621784547999994 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174030", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102725400000168, + "build_seconds": 11.179411349000006, + "workflow_seconds": null, + "total_seconds": 11.280438603000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0027033590000087315, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.280438603000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174046", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20137798899999382, + "build_seconds": 26.721995090999997, + "workflow_seconds": null, + "total_seconds": 26.92337307999999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3949275660000069, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 26.92337307999999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174029", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] + }, + { + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135873500001367, + "build_seconds": 12.511644949000015, + "workflow_seconds": null, + "total_seconds": 12.713003684000029, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.34990612199999305, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 12.713003684000029 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174045", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] + } + ], + "exclusions": [] +} diff --git a/results/stackstorm-pants/registry-screening-04/report.md b/results/stackstorm-pants/registry-screening-04/report.md new file mode 100644 index 000000000..e336c3137 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/report.md @@ -0,0 +1,33 @@ +# stackstorm-pants: registry-screening-04 + +Question: Does Pants restore StackStorm pack metadata test results on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 4/4 recorded; 0 failed; 0 missing. + +Completion checks: passed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | bazel-remote | 1 | unmeasured | 0 | +| Warm build | bazel-remote | 1 | unmeasured | 0 | +| Cold build | BoringCache | 1 | unmeasured | 0 | +| Warm build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | bazel-remote | Cold build | 0.1010307130000001 | 27.520753834999994 | 27.621784547999994 | unmeasured | unmeasured | miss | [JSON](runs/1-bazel-remote-cold.json) | +| 1 | bazel-remote | Warm build | 0.10102725400000168 | 11.179411349000006 | 11.280438603000007 | unmeasured | unmeasured | hit | [JSON](runs/1-bazel-remote-warm.json) | +| 1 | BoringCache | Cold build | 0.20137798899999382 | 26.721995090999997 | 26.92337307999999 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | BoringCache | Warm build | 0.20135873500001367 | 12.511644949000015 | 12.713003684000029 | unmeasured | unmeasured | hit | [JSON](runs/1-boringcache-warm.json) | + +[Full records and checks](report.json) diff --git a/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-cold.json b/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-cold.json new file mode 100644 index 000000000..069fa2868 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-cold.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1010307130000001, + "build_seconds": 27.520753834999994, + "workflow_seconds": null, + "total_seconds": 27.621784547999994, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0030729709999945953, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 27.621784547999994 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174030", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] +} diff --git a/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-warm.json b/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-warm.json new file mode 100644 index 000000000..a93e812fc --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/runs/1-bazel-remote-warm.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "bazel-remote", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.10102725400000168, + "build_seconds": 11.179411349000006, + "workflow_seconds": null, + "total_seconds": 11.280438603000007, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.0027033590000087315, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 11.280438603000007 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174046", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] +} diff --git a/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-cold.json b/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-cold.json new file mode 100644 index 000000000..65c54307a --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-cold.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20137798899999382, + "build_seconds": 26.721995090999997, + "workflow_seconds": null, + "total_seconds": 26.92337307999999, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.3949275660000069, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 26.92337307999999 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174029", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] +} diff --git a/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-warm.json b/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-warm.json new file mode 100644 index 000000000..ba7955df4 --- /dev/null +++ b/results/stackstorm-pants/registry-screening-04/runs/1-boringcache-warm.json @@ -0,0 +1,141 @@ +{ + "schema_version": 1, + "benchmark": "stackstorm-pants", + "strategy": "boringcache", + "lane": "fresh", + "phase": "warm", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "stackstorm-pants", + "workspace": "boringcache/benchmarks", + "definition_sha256": "079f470036285d650884970a4b29382e9c831ba7cb154379451a80e1b39fd2e5", + "source": { + "repository": "StackStorm/st2", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + } + ] + }, + "origin": { + "repository": "StackStorm/st2", + "branch": "master", + "revision": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Pants restore StackStorm pack metadata test results on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "stackstorm-pants" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.20135873500001367, + "build_seconds": 12.511644949000015, + "workflow_seconds": null, + "total_seconds": 12.713003684000029, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.34990612199999305, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 12.713003684000029 + }, + "cache": { + "hit": true, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "stackstorm-pants-series-registry-screening-04-s1-r37302896058-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "StackStorm/st2", + "sha": "9824de4dfd0c869869e310dee729308f398ad83a" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37302896058", + "run_attempt": "1", + "job": "warm", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174045", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37302896058-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37302896058" + ] +} From 9891872472d550d35a6ff0df77b0d9aa4e306a34 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:42:35 +0100 Subject: [PATCH 14/22] Link the completed native cache repeat results --- cases/gogs-moon/proposal.md | 3 ++- cases/msgpack-sbt/proposal.md | 3 ++- cases/opencut-moon/proposal.md | 3 ++- cases/stackstorm-pants/proposal.md | 3 ++- docs/reapi-evaluations.md | 18 ++++++++++++------ docs/tool-coverage.md | 14 ++++++++------ 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/cases/gogs-moon/proposal.md b/cases/gogs-moon/proposal.md index 12be37143..d71c3d743 100644 --- a/cases/gogs-moon/proposal.md +++ b/cases/gogs-moon/proposal.md @@ -1,6 +1,7 @@ # gogs-moon -Status: native REAPI correctness screen implemented; hosted qualification pending. +Status: two cold/warm samples passed for both providers. +[Recorded results](../../results/gogs-moon/registry-qualification-01/report.md). Source: [gogs/gogs at `dbbd717e923694c36f41b0360515b4374bbe6ee0`](https://github.com/gogs/gogs/tree/dbbd717e923694c36f41b0360515b4374bbe6ee0). diff --git a/cases/msgpack-sbt/proposal.md b/cases/msgpack-sbt/proposal.md index 4c13a99b2..90d2fcdef 100644 --- a/cases/msgpack-sbt/proposal.md +++ b/cases/msgpack-sbt/proposal.md @@ -1,6 +1,7 @@ # msgpack-sbt -Status: native REAPI correctness screen implemented; hosted qualification pending. +Status: two cold/warm samples passed for both providers. +[Recorded results](../../results/msgpack-sbt/registry-qualification-01/report.md). Source: [msgpack/msgpack-java at `5c28fbad6cd360d8ea822df109830ae2517a7ae4`](https://github.com/msgpack/msgpack-java/tree/5c28fbad6cd360d8ea822df109830ae2517a7ae4). diff --git a/cases/opencut-moon/proposal.md b/cases/opencut-moon/proposal.md index 079ce9d8e..53be2096f 100644 --- a/cases/opencut-moon/proposal.md +++ b/cases/opencut-moon/proposal.md @@ -1,6 +1,7 @@ # opencut-moon -Status: native REAPI correctness screen implemented; hosted qualification pending. +Status: two cold/warm samples passed for both providers. +[Recorded results](../../results/opencut-moon/registry-qualification-01/report.md). The benchmark uses Moon 2.5.6 instead of upstream's 2.3.3. The older client rejects a registry that advertises disabled action-cache updates, including a read-only diff --git a/cases/stackstorm-pants/proposal.md b/cases/stackstorm-pants/proposal.md index d3911d471..47d2b22f2 100644 --- a/cases/stackstorm-pants/proposal.md +++ b/cases/stackstorm-pants/proposal.md @@ -1,6 +1,7 @@ # stackstorm-pants -Status: native REAPI correctness screen implemented; hosted qualification pending. +Status: two cold/warm samples passed for both providers. +[Recorded results](../../results/stackstorm-pants/registry-qualification-01/report.md). Source: [StackStorm/st2 at `9824de4dfd0c869869e310dee729308f398ad83a`](https://github.com/StackStorm/st2/tree/9824de4dfd0c869869e310dee729308f398ad83a). diff --git a/docs/reapi-evaluations.md b/docs/reapi-evaluations.md index b5204237f..c4d96ea8b 100644 --- a/docs/reapi-evaluations.md +++ b/docs/reapi-evaluations.md @@ -1,14 +1,15 @@ # Native gRPC cache evaluations -These eight source-pinned evaluations use the existing case contract. Five have -a shared native registry workflow awaiting hosted qualification. ZITADEL, Pants +These eight source-pinned evaluations use the existing case contract. Five use +the shared native registry workflow. Moon, Pants and sbt have completed two +cold/warm samples per selected workload; Buck2 qualification is running. ZITADEL, Pants JVM and Buck2 Prelude remain blocked drafts. None has an active schedule. | Case | Workload | Reason to evaluate | Limitation | | --- | --- | --- | --- | -| [opencut-moon](../cases/opencut-moon/proposal.md) | OpenCut web assets | Existing Moon build and CI graph | Qualify upstream Moon 2.3.3 and native remote output restoration | +| [opencut-moon](../cases/opencut-moon/proposal.md) | OpenCut web assets | Existing Moon build and CI graph | Moon 2.5.6 replaces upstream 2.3.3; web assets only | | [gogs-moon](../cases/gogs-moon/proposal.md) | Gogs frontend | Upstream Moon web build | Web assets only; Go server compilation is outside this screen | -| [executorch-buck2](../cases/executorch-buck2/proposal.md) | ExecuTorch portable executor | Native C++ runtime and kernel compilation | Verify the public target with upstream Buck2 2025-05-06 and pinned submodules | +| [executorch-buck2](../cases/executorch-buck2/proposal.md) | ExecuTorch portable executor | Native C++ runtime and kernel compilation | Buck2 2025-06-01 replaces upstream 2025-05-06; executable checks do not run a model | | [stackstorm-pants](../cases/stackstorm-pants/proposal.md) | StackStorm plugin tests | Existing Pants test CI job | Pack metadata unit tests only; service-dependent tests are outside this screen | | [zitadel-moon](../cases/zitadel-moon/proposal.md) | ZITADEL console assets | Existing Moon task graph and remote-cache configuration | Resolve its lockfile client version and record the instance-name change | | [pants-jvm](../cases/pants-jvm/proposal.md) | Pants Java/Scala example | Native JVM compile, test, and package operations | Small correctness screen; upstream client differs from product qualification | @@ -48,7 +49,12 @@ and whether uploads complete inside or outside the timed command. Follow OpenCut, Gogs, StackStorm, ExecuTorch and MessagePack now declare the shared native registry correctness workflow. Their proposals record selected targets, -recipe changes and verification requirements. Hosted qualification remains -pending. BoringCache uses a fresh remote tag and a read-only warm process; +recipe changes and verification requirements. The two-sample reports for +[Gogs](../results/gogs-moon/registry-qualification-01/report.md), +[OpenCut](../results/opencut-moon/registry-qualification-01/report.md), +[StackStorm](../results/stackstorm-pants/registry-qualification-01/report.md) and +[MessagePack](../results/msgpack-sbt/registry-qualification-01/report.md) retain +each provider and phase. Buck2 qualification remains pending. +BoringCache uses a fresh remote tag and a read-only warm process; bazel-remote transfers its store as an artifact outside timing. No performance comparison or completed family qualification follows from implementation alone. diff --git a/docs/tool-coverage.md b/docs/tool-coverage.md index 7e1c73d26..4a23d28ae 100644 --- a/docs/tool-coverage.md +++ b/docs/tool-coverage.md @@ -18,11 +18,11 @@ limitations. This inventory describes the case definitions as of October 5, 2026 | Nx remote cache | No active case | Storybook archives Nx directories; it does not exercise the native Nx remote-cache adapter | | Turbo | n8n; PostHog plan | Qualify each selected workload and timing boundary | | Bazel | gRPC C++ client and server | [One three-provider cold/warm sample passed](../results/grpc/local-outputs-01/report.md), including local output checks; BuildBuddy storage remains unmeasured; publication and rolling qualification remain open | -| Moon | [OpenCut](../cases/opencut-moon/proposal.md), [Gogs](../cases/gogs-moon/proposal.md), ZITADEL drafts | Qualify the pinned clients, remote task restoration and output checks | -| Pants | [StackStorm](../cases/stackstorm-pants/proposal.md), Pants JVM drafts | Qualify the pinned clients; review service-dependent test cacheability | +| Moon | [OpenCut](../cases/opencut-moon/proposal.md), [Gogs](../cases/gogs-moon/proposal.md), ZITADEL drafts | OpenCut and Gogs passed two native cold/warm samples each; ZITADEL remains a draft | +| Pants | [StackStorm](../cases/stackstorm-pants/proposal.md), Pants JVM drafts | StackStorm plugin tests passed two cold/warm samples; service-dependent tests and Pants JVM remain outside this qualification | | Buck2 | [ExecuTorch](../cases/executorch-buck2/proposal.md), Buck2 Prelude drafts | Verify public targets, native action-cache use and output checks | -| sbt | [MessagePack Java](../cases/msgpack-sbt/proposal.md) draft | Qualify sbt 2 remote-cache configuration and compiled outputs | -| Nix | [Zed](../cases/zed-nix/proposal.md), [Helix](../cases/helix-nix/proposal.md) | Shared Nix workflow implemented; cold/warm qualification pending | +| sbt | [MessagePack Java](../cases/msgpack-sbt/proposal.md) | Two native cold/warm samples passed; managed product adapter remains separate | +| Nix | [Zed](../cases/zed-nix/proposal.md), [Helix](../cases/helix-nix/proposal.md) | Helix cold/warm screen passed; repeat checks and Zed qualification are running | | Archive | Storybook's Nx cache directories | Qualify rolling source and seed lineage | | Artifact, GHA compatibility service | No dedicated active comparison case | GitHub artifact uploads inside a benchmark do not qualify these BoringCache products | @@ -31,9 +31,11 @@ benchmark adapter. Product support includes Bazel, ccache, Go, Gradle, Maven, Nx sccache, and Turbo; this repository does not yet qualify every injected family. Cargo target mounts use the product's managed freshness behavior and can compose sccache. Nix's daemon/store and macOS Xcode are not Linux Docker injection modes. -The selected [REAPI cases](reapi-evaluations.md) use one shared native registry workflow. Hosted correctness qualification is pending; the remaining drafts stay blocked. +The selected [REAPI cases](reapi-evaluations.md) use one shared native registry workflow. +Moon, Pants and sbt have retained two-sample correctness results. Buck2 +qualification is running; the remaining drafts stay blocked. The Nix cases use one shared fresh workflow and the public Nix product mode; -qualification remains pending. The Cachix cache name and token are repository +Helix has retained screening results. The Cachix cache name and token are repository configuration. No new schedules are enabled. Each family has a pinned OSS workload; definitions and workflows do not establish measured coverage. MessagePack keeps its existing case ID and pin. From 2a7352597935f9089b8a13dcbcb0a3a6767cd844 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:43:50 +0100 Subject: [PATCH 15/22] Pair Buck2 with its released Prelude commit --- .../payload/reapi-recipe.json | 18 +++++++ cases/executorch-buck2/proposal.md | 5 +- .../registry-screening-06/series.json | 49 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 results/executorch-buck2/registry-screening-06/series.json diff --git a/cases/executorch-buck2/payload/reapi-recipe.json b/cases/executorch-buck2/payload/reapi-recipe.json index 8b2646b56..b3a8ba971 100644 --- a/cases/executorch-buck2/payload/reapi-recipe.json +++ b/cases/executorch-buck2/payload/reapi-recipe.json @@ -14,6 +14,24 @@ "--init", "--recursive" ], + [ + "git", + "-C", + "third-party/prelude", + "fetch", + "--depth", + "1", + "origin", + "cb8b34f660977893418853c0001b319cc9314798" + ], + [ + "git", + "-C", + "third-party/prelude", + "checkout", + "--detach", + "cb8b34f660977893418853c0001b319cc9314798" + ], [ "python3", "-m", diff --git a/cases/executorch-buck2/proposal.md b/cases/executorch-buck2/proposal.md index eb67595ba..b47b14967 100644 --- a/cases/executorch-buck2/proposal.md +++ b/cases/executorch-buck2/proposal.md @@ -15,7 +15,10 @@ Buck2 2025-06-01 instead of the upstream 2025-05-06 pin. The newer client includ the [permission-check digest fix](https://github.com/facebook/buck2/pull/946); the old client sends SHA-1 blobs to the SHA-256 cache during its upload check. The payload adds a local execution platform with remote caching enabled and -remote execution disabled. Source submodules retain upstream pins. Verification +remote execution disabled. The Prelude submodule uses +`cb8b34f660977893418853c0001b319cc9314798`, the commit published in the +Buck2 2025-06-01 release’s `prelude_hash` asset. Other submodules retain upstream +pins. Verification checks executable ELF output and cold/warm hashes; it does not run an exported model. Declared command: `buck2 build //examples/portable/executor_runner:executor_runner --show-output`. diff --git a/results/executorch-buck2/registry-screening-06/series.json b/results/executorch-buck2/registry-screening-06/series.json new file mode 100644 index 000000000..2336403b1 --- /dev/null +++ b/results/executorch-buck2/registry-screening-06/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-06", + "created_at": "2026-10-05T11:43:50Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 1, + "definition_sha256": "3cd44619ccb92779907808414e3ebd5257214cd68e324bea16a2cf7a3b6ee6c3", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "173938e78307c1da76ab31da0b375e31949d07113e431d885fc2f44dfb30711f" +} From f2ca2ed5561491c873b5c6e7037b3ddf7989d34b Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:48:11 +0100 Subject: [PATCH 16/22] Retain Buck2 compatibility attempts and Nix screen status --- cases/helix-nix/proposal.md | 5 +- data/latest/series.json | 185 +++++++++++++-- migration/evidence.json | 28 +++ .../completions/37303662752.json | 16 ++ .../registry-screening-04/report.json | 223 ++++++++++++++++++ .../registry-screening-04/report.md | 38 +++ .../runs/1-boringcache-cold.json | 142 +++++++++++ .../completions/37304358750.json | 17 ++ .../registry-screening-05/report.json | 78 ++++++ .../registry-screening-05/report.md | 38 +++ .../1-reapi-fresh-benchmark.yml.json | 36 +++ 11 files changed, 788 insertions(+), 18 deletions(-) create mode 100644 results/executorch-buck2/registry-screening-04/completions/37303662752.json create mode 100644 results/executorch-buck2/registry-screening-04/report.json create mode 100644 results/executorch-buck2/registry-screening-04/report.md create mode 100644 results/executorch-buck2/registry-screening-04/runs/1-boringcache-cold.json create mode 100644 results/executorch-buck2/registry-screening-05/completions/37304358750.json create mode 100644 results/executorch-buck2/registry-screening-05/report.json create mode 100644 results/executorch-buck2/registry-screening-05/report.md create mode 100644 results/executorch-buck2/registry-screening-06/dispatches/1-reapi-fresh-benchmark.yml.json diff --git a/cases/helix-nix/proposal.md b/cases/helix-nix/proposal.md index e1bc68e06..aa6ce64fb 100644 --- a/cases/helix-nix/proposal.md +++ b/cases/helix-nix/proposal.md @@ -1,7 +1,8 @@ # Helix / Nix -Status: workflow implemented; cold/warm qualification is pending. -No benchmark measurements have been collected. +Status: the cold/warm screen passed for both providers, including matching NAR +hashes. [Recorded results](../../results/helix-nix/canary-screening-02/report.md). +The two-sample repeat is running. Source: [helix-editor/helix at `ba40e547426b0f9896c8bdc699a4ab11f2b37dbc`](https://github.com/helix-editor/helix/tree/ba40e547426b0f9896c8bdc699a4ab11f2b37dbc). diff --git a/data/latest/series.json b/data/latest/series.json index 42c691edb..1fed2feec 100644 --- a/data/latest/series.json +++ b/data/latest/series.json @@ -2755,7 +2755,7 @@ }, "plan_sha256": "b8d203ceb51e1632633cc43c486fd493c4effe513931a84e984cbebcad7c6257", "definition_sha256": "5ce3b7d62703030a03f65e84bd843034d8faa022481a212d964c3dfbb85931b7", - "state": "requested", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -2763,11 +2763,6 @@ "This series does not declare a comparison of provider performance." ], "missing_observations": [ - [ - 1, - "cold", - "boringcache" - ], [ 1, "cold", @@ -2784,7 +2779,16 @@ "bazel-remote" ] ], - "summaries": [], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], "dispatches": [ { "case_id": "executorch-buck2", @@ -2804,13 +2808,42 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752" } ], - "completions": [], - "report_state": "missing", - "report_path": null, - "evidence": [], - "runs_without_archives": [ - "37303662752" - ] + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37303662752", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "a5fe015908b1dc7a64b528c76d66c185c309a692d900f26c405b1292992d99e6" + } + ], + "report_state": "current", + "report_path": "results/executorch-buck2/registry-screening-04/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37303662752, + "inventory_state": "complete", + "verified_files": 10, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303662752.tar.gz", + "bundle_bytes": 237157, + "sha256": "ec972a7c4b1aa68a041352068696ab4db8a44ed5604da057e12dbbfe2ed6c9a3", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] }, { "case_id": "executorch-buck2", @@ -2845,7 +2878,7 @@ }, "plan_sha256": "10fca1a272ec3569d72de5bc0ed5a687263e999a856cbd9181cd3dddca073f25", "definition_sha256": "a5fcff27c94f6fb115ea5aff07d0cbb1b571d30f7966c7ada60700a2e22b66f2", - "state": "requested", + "state": "failed", "publication": "unreviewed", "valid_for_comparison": false, "execution_verified": false, @@ -2894,12 +2927,132 @@ "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750" } ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "sample": 1, + "run_id": "37304358750", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750", + "source_sha": "285eb0d0ad866c46e67e616f6210a7906dcdab9f", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "e18047f4b27a0c5572b24b1a4b25206793e258d2a92e05b4267f5b14ae52c6c0" + } + ], + "report_state": "current", + "report_path": "results/executorch-buck2/registry-screening-05/report.json", + "evidence": [ + { + "repository": "boringcache/benchmarks", + "run_id": 37304358750, + "inventory_state": "complete", + "verified_files": 9, + "gaps": [], + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37304358750.tar.gz", + "bundle_bytes": 72525, + "sha256": "8b69365e4ef2fcafc79b52ff3c84d5c1cfd15231eeddf067c453d7f0ae3bc1df", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory" + } + ], + "runs_without_archives": [] + }, + { + "case_id": "executorch-buck2", + "series_id": "registry-screening-06", + "plan_path": "results/executorch-buck2/registry-screening-06/series.json", + "lane": "fresh", + "variant": null, + "sample_count": 1, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "plan_sha256": "173938e78307c1da76ab31da0b375e31949d07113e431d885fc2f44dfb30711f", + "definition_sha256": "3cd44619ccb92779907808414e3ebd5257214cd68e324bea16a2cf7a3b6ee6c3", + "state": "requested", + "publication": "unreviewed", + "valid_for_comparison": false, + "execution_verified": false, + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "missing_observations": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "dispatches": [ + { + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-06", + "sample": "1" + }, + "ref": "qualify-native-registry", + "state": "requested", + "requested_at": "2026-10-05T11:44:04Z", + "run_id": 37304834734, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304834734" + } + ], "completions": [], "report_state": "missing", "report_path": null, "evidence": [], "runs_without_archives": [ - "37304358750" + "37304834734" ] }, { diff --git a/migration/evidence.json b/migration/evidence.json index 5c4c68284..9e4f37066 100644 --- a/migration/evidence.json +++ b/migration/evidence.json @@ -1555,6 +1555,34 @@ "scope": "One gRPC cold/warm sample across three providers, including attempts, jobs, logs, artifacts, commit and workflow", "execution_qualification": "Six output checks and BoringCache post steps passed; one sample, BuildBuddy storage unmeasured, publication unreviewed", "repository_retirement": "This run does not approve retirement of any original repository" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37303662752, + "verified_files": 10, + "gaps": [], + "sha256": "ec972a7c4b1aa68a041352068696ab4db8a44ed5604da057e12dbbfe2ed6c9a3", + "bundle_bytes": 237157, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37303662752.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" + }, + { + "repository": "boringcache/benchmarks", + "run_id": 37304358750, + "verified_files": 9, + "gaps": [], + "sha256": "8b69365e4ef2fcafc79b52ff3c84d5c1cfd15231eeddf067c453d7f0ae3bc1df", + "bundle_bytes": 72525, + "inventory_state": "complete", + "release_url": "https://github.com/boringcache/benchmarks/releases/tag/evidence-native-registry-2026-10-05", + "bundle_url": "https://github.com/boringcache/benchmarks/releases/download/evidence-native-registry-2026-10-05/benchmarks-37304358750.tar.gz", + "publication_verification": "Downloaded the published archive again, matched its SHA-256, and verified all listed files and the run inventory", + "verified_on": "2026-10-05", + "scope": "This run, including available attempts, jobs, logs, artifacts, commit and workflow" } ] } diff --git a/results/executorch-buck2/registry-screening-04/completions/37303662752.json b/results/executorch-buck2/registry-screening-04/completions/37303662752.json new file mode 100644 index 000000000..fd5e79161 --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/completions/37303662752.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37303662752", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "a5fe015908b1dc7a64b528c76d66c185c309a692d900f26c405b1292992d99e6" +} diff --git a/results/executorch-buck2/registry-screening-04/report.json b/results/executorch-buck2/registry-screening-04/report.json new file mode 100644 index 000000000..0032e6283 --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/report.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "plan_sha256": "b8d203ceb51e1632633cc43c486fd493c4effe513931a84e984cbebcad7c6257", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37303662752", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "a5fe015908b1dc7a64b528c76d66c185c309a692d900f26c405b1292992d99e6" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-04", + "sample": 1, + "run_id": "37303662752", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37303662752", + "source_sha": "c99edae5024cbe6e0f845f4b3e34fc691c671bc9", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "a5fe015908b1dc7a64b528c76d66c185c309a692d900f26c405b1292992d99e6" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": null, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ce3b7d62703030a03f65e84bd843034d8faa022481a212d964c3dfbb85931b7", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009217370000215, + "build_seconds": 153.421008911, + "workflow_seconds": null, + "total_seconds": 153.52193064800002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.20521032599998534, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 153.52193064800002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-series-registry-screening-04-s1-r37303662752-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303662752", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174079", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303662752-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303662752" + ] + } + ], + "exclusions": [] +} diff --git a/results/executorch-buck2/registry-screening-04/report.md b/results/executorch-buck2/registry-screening-04/report.md new file mode 100644 index 000000000..50670dcba --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: registry-screening-04 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 1/4 recorded; 0 failed; 3 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | +| Cold build | BoringCache | 1 | unmeasured | 0 | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37303662752 failed completion checks: + +- Workflow concluded failure +- bazel-remote executorch-buck2 cold: failure + +Missing observations: + +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 0.1009217370000215 | 153.421008911 | 153.52193064800002 | unmeasured | unmeasured | not reported | [JSON](runs/1-boringcache-cold.json) | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/registry-screening-04/runs/1-boringcache-cold.json b/results/executorch-buck2/registry-screening-04/runs/1-boringcache-cold.json new file mode 100644 index 000000000..d5e0ee58f --- /dev/null +++ b/results/executorch-buck2/registry-screening-04/runs/1-boringcache-cold.json @@ -0,0 +1,142 @@ +{ + "schema_version": 1, + "benchmark": "executorch-buck2", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "reapi", + "adapter": "runtime", + "case": { + "schema_version": 1, + "case_id": "executorch-buck2", + "workspace": "boringcache/benchmarks", + "definition_sha256": "5ce3b7d62703030a03f65e84bd843034d8faa022481a212d964c3dfbb85931b7", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "origin": { + "repository": "pytorch/executorch", + "branch": "main", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/reapi-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2" + } + } + ], + "blockers": [] + } + }, + "timing": { + "restore_or_setup_seconds": 0.1009217370000215, + "build_seconds": 153.421008911, + "workflow_seconds": null, + "total_seconds": 153.52193064800002, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": 0.20521032599998534, + "comparison_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "build_and_reuse_seconds": 153.52193064800002 + }, + "cache": { + "hit": null, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "executorch-buck2-series-registry-screening-04-s1-r37303662752-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "pytorch/executorch", + "sha": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + }, + "product_refs": { + "cli_version": "1.34.0" + }, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37303662752", + "run_attempt": "1", + "job": "cold", + "workflow": "Native REAPI screening", + "ref_name": "qualify-native-registry", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000174079", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37303662752-1", + "series": { + "id": "registry-screening-04", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37303662752" + ] +} diff --git a/results/executorch-buck2/registry-screening-05/completions/37304358750.json b/results/executorch-buck2/registry-screening-05/completions/37304358750.json new file mode 100644 index 000000000..c0de4df27 --- /dev/null +++ b/results/executorch-buck2/registry-screening-05/completions/37304358750.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "sample": 1, + "run_id": "37304358750", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750", + "source_sha": "285eb0d0ad866c46e67e616f6210a7906dcdab9f", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "e18047f4b27a0c5572b24b1a4b25206793e258d2a92e05b4267f5b14ae52c6c0" +} diff --git a/results/executorch-buck2/registry-screening-05/report.json b/results/executorch-buck2/registry-screening-05/report.json new file mode 100644 index 000000000..55a5e668e --- /dev/null +++ b/results/executorch-buck2/registry-screening-05/report.json @@ -0,0 +1,78 @@ +{ + "schema_version": 1, + "plan_sha256": "10fca1a272ec3569d72de5bc0ed5a687263e999a856cbd9181cd3dddca073f25", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [ + "This series does not declare a comparison of provider performance." + ], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "sample": 1, + "run_id": "37304358750", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750", + "source_sha": "285eb0d0ad866c46e67e616f6210a7906dcdab9f", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "e18047f4b27a0c5572b24b1a4b25206793e258d2a92e05b4267f5b14ae52c6c0" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-screening-05", + "sample": 1, + "run_id": "37304358750", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304358750", + "source_sha": "285eb0d0ad866c46e67e616f6210a7906dcdab9f", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "bazel-remote executorch-buck2 cold: failure", + "boringcache executorch-buck2 cold: failure" + ], + "evidence_manifest_sha256": "e18047f4b27a0c5572b24b1a4b25206793e258d2a92e05b4267f5b14ae52c6c0" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "correctness", + "missing": [ + [ + 1, + "cold", + "boringcache" + ], + [ + 1, + "cold", + "bazel-remote" + ], + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "bazel-remote" + ] + ], + "summaries": [], + "records": [], + "exclusions": [] +} diff --git a/results/executorch-buck2/registry-screening-05/report.md b/results/executorch-buck2/registry-screening-05/report.md new file mode 100644 index 000000000..1ae3262aa --- /dev/null +++ b/results/executorch-buck2/registry-screening-05/report.md @@ -0,0 +1,38 @@ +# executorch-buck2: registry-screening-05 + +Question: Does Buck2 restore the ExecuTorch portable runner on a clean worker? + +Measured scope: Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. + +Observations: 0/4 recorded; 0 failed; 4 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | ---: | + +Comparison checks: + +- This series does not declare a comparison of provider performance. + +Run 37304358750 failed completion checks: + +- Workflow concluded failure +- bazel-remote executorch-buck2 cold: failure +- boringcache executorch-buck2 cold: failure + +Missing observations: + +- Sample 1, BoringCache, Cold build +- Sample 1, bazel-remote, Cold build +- Sample 1, BoringCache, Warm build +- Sample 1, bazel-remote, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | + +[Full records and checks](report.json) diff --git a/results/executorch-buck2/registry-screening-06/dispatches/1-reapi-fresh-benchmark.yml.json b/results/executorch-buck2/registry-screening-06/dispatches/1-reapi-fresh-benchmark.yml.json new file mode 100644 index 000000000..5fa17298e --- /dev/null +++ b/results/executorch-buck2/registry-screening-06/dispatches/1-reapi-fresh-benchmark.yml.json @@ -0,0 +1,36 @@ +{ + "case_id": "executorch-buck2", + "repository": "boringcache/benchmarks", + "workflow": "reapi-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "lane": "fresh", + "inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe", + "series_id": "registry-screening-06", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T11:44:04Z", + "ref": "qualify-native-registry", + "state": "requested", + "run_id": 37304834734, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37304834734" +} From 9571a4dc1b8b6474f4d1071803ce7e076e41ab7e Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 12:50:07 +0100 Subject: [PATCH 17/22] Declare repeated Buck2 cache checks --- .../registry-qualification-01/series.json | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 results/executorch-buck2/registry-qualification-01/series.json diff --git a/results/executorch-buck2/registry-qualification-01/series.json b/results/executorch-buck2/registry-qualification-01/series.json new file mode 100644 index 000000000..253475e0c --- /dev/null +++ b/results/executorch-buck2/registry-qualification-01/series.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "case_id": "executorch-buck2", + "series_id": "registry-qualification-01", + "created_at": "2026-10-05T11:50:07Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "3cd44619ccb92779907808414e3ebd5257214cd68e324bea16a2cf7a3b6ee6c3", + "variant": null, + "workflow_inputs": { + "case_id": "executorch-buck2", + "cli_version": "vcli-canary-7a5b27146ebe" + }, + "comparison": { + "question": "Does Buck2 restore the ExecuTorch portable runner on a clean worker?", + "providers": [ + "boringcache", + "bazel-remote" + ], + "primary_metric": "correctness", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "storage": "unmeasured", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "sample_count": 2 + }, + "source": { + "repository": "pytorch/executorch", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" + } + ], + "patch": "cache.patch" + }, + "verification": [ + "Require the declared native command to succeed", + "Require declared outputs and successful output checks", + "Require native remote-hit evidence and the same output hashes on the warm worker", + "Require registry shutdown to complete successfully" + ], + "plan_sha256": "f63d3cd794609256c2260642e2fb62087c1c0d96eb6e41e9f7ae440048d137f0" +} From 81fc713d2a201d6e4c4f064e7dcd42cb960f2b95 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 13:18:45 +0100 Subject: [PATCH 18/22] Use one suite for fresh schedules and source checks --- .github/workflows/canary.yml | 35 +++------ .github/workflows/fresh-cadence.yml | 47 +++++++++++ .github/workflows/guardrails.yml | 2 + .github/workflows/nightly-canaries.yml | 3 +- .github/workflows/source-sync.yml | 6 +- .github/workflows/weekly-fresh.yml | 27 +++++++ docs/cadence.md | 48 ++++++++++++ docs/process.md | 3 + scripts/benchmark-cadence.rb | 71 +++++++++++++++++ scripts/check-registry-alignment.rb | 3 +- scripts/nightly-canaries.rb | 39 +++++++--- scripts/sync-sources.rb | 51 ++++++++---- suites/scheduled.json | 103 +++++++++++++++++++++++++ test/benchmark_cadence_test.rb | 94 ++++++++++++++++++++++ 14 files changed, 478 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/fresh-cadence.yml create mode 100644 .github/workflows/weekly-fresh.yml create mode 100644 docs/cadence.md create mode 100644 scripts/benchmark-cadence.rb create mode 100644 suites/scheduled.json create mode 100644 test/benchmark_cadence_test.rb diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 47c62d872..f151af8a1 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -1,38 +1,27 @@ name: Dispatch CLI Canary Cases on: + schedule: + - cron: '17 1 * * *' workflow_dispatch: inputs: cli_version: description: Optional exact published CLI canary; empty selects the latest type: string default: '' + dry_run: + description: Validate the dispatch without starting builds. + type: boolean + default: false permissions: contents: read + actions: write concurrency: group: benchmark-canary-dispatch cancel-in-progress: false jobs: dispatch: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - permissions: - contents: read - actions: write - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - name: Dispatch registered cases - env: - GH_TOKEN: ${{ github.token }} - CLI_VERSION: ${{ inputs.cli_version }} - run: ruby scripts/nightly-canaries.rb --repository "$GITHUB_REPOSITORY" --version "$CLI_VERSION" --output "$RUNNER_TEMP/nightly-canaries.json" --summary "$GITHUB_STEP_SUMMARY" - - name: Retain dispatch receipt - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: nightly-canaries - path: ${{ runner.temp }}/nightly-canaries.json - if-no-files-found: error - retention-days: 90 + uses: ./.github/workflows/fresh-cadence.yml + with: + channel: canary + cli_version: ${{ inputs.cli_version || '' }} + dry_run: ${{ inputs.dry_run || false }} diff --git a/.github/workflows/fresh-cadence.yml b/.github/workflows/fresh-cadence.yml new file mode 100644 index 000000000..cf0be3fd9 --- /dev/null +++ b/.github/workflows/fresh-cadence.yml @@ -0,0 +1,47 @@ +name: Dispatch fresh benchmark runs +on: + workflow_call: + inputs: + channel: + type: string + required: true + cli_version: + type: string + default: '' + dry_run: + type: boolean + default: false +permissions: + contents: read + actions: write +jobs: + dispatch: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - name: Dispatch the scheduled suite + env: + GH_TOKEN: ${{ github.token }} + CHANNEL: ${{ inputs.channel }} + CLI_VERSION: ${{ inputs.cli_version }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + args=() + [[ "$DRY_RUN" != true ]] || args+=(--dry-run) + bundle exec ruby scripts/benchmark-cadence.rb \ + --channel "$CHANNEL" --version "$CLI_VERSION" --ref "$GITHUB_REF_NAME" \ + --output "$RUNNER_TEMP/benchmark-dispatch.json" \ + --summary "$GITHUB_STEP_SUMMARY" "${args[@]}" + - name: Retain dispatch receipt + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-dispatch + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/guardrails.yml b/.github/workflows/guardrails.yml index 82a82f5c6..4d8d0d2fe 100644 --- a/.github/workflows/guardrails.yml +++ b/.github/workflows/guardrails.yml @@ -16,6 +16,8 @@ jobs: run: bundle exec ruby bin/bench check - name: Check publication registry run: bundle exec ruby scripts/check-registry-alignment.rb + - name: Check scheduled cases + run: bundle exec ruby scripts/benchmark-cadence.rb --check - name: Check execution contracts run: bundle exec ruby scripts/check-workflow-guardrails.rb - name: Check report contracts diff --git a/.github/workflows/nightly-canaries.yml b/.github/workflows/nightly-canaries.yml index 3a71039bc..4d24f63a0 100644 --- a/.github/workflows/nightly-canaries.yml +++ b/.github/workflows/nightly-canaries.yml @@ -17,8 +17,9 @@ jobs: with: persist-credentials: false - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 - name: Check repository canaries - run: ruby scripts/nightly-canaries.rb --collect --summary "${GITHUB_STEP_SUMMARY}" + run: bundle exec ruby scripts/benchmark-cadence.rb --collect --summary "${GITHUB_STEP_SUMMARY}" 'on': schedule: - cron: 47 * * * * diff --git a/.github/workflows/source-sync.yml b/.github/workflows/source-sync.yml index cfe31321a..72fe62a05 100644 --- a/.github/workflows/source-sync.yml +++ b/.github/workflows/source-sync.yml @@ -1,9 +1,11 @@ -name: Propose Upstream Source Updates +name: Check upstream source updates on: + schedule: + - cron: '11 0 * * *' workflow_dispatch: inputs: case_id: - description: Optional case ID; empty inspects all maintained source schedules + description: Optional case ID; empty checks the scheduled suite. type: string default: '' permissions: diff --git a/.github/workflows/weekly-fresh.yml b/.github/workflows/weekly-fresh.yml new file mode 100644 index 000000000..29e7197d5 --- /dev/null +++ b/.github/workflows/weekly-fresh.yml @@ -0,0 +1,27 @@ +name: Weekly fresh benchmarks +on: + schedule: + - cron: '0 4 * * 1' + workflow_dispatch: + inputs: + cli_version: + description: Optional exact stable CLI release; empty selects the latest. + type: string + default: '' + dry_run: + description: Validate the dispatch without starting builds. + type: boolean + default: false +permissions: + contents: read + actions: write +jobs: + dispatch: + concurrency: + group: benchmark-weekly-dispatch + cancel-in-progress: false + uses: ./.github/workflows/fresh-cadence.yml + with: + channel: stable + cli_version: ${{ inputs.cli_version || '' }} + dry_run: ${{ inputs.dry_run || false }} diff --git a/docs/cadence.md b/docs/cadence.md new file mode 100644 index 000000000..8d5a1d89d --- /dev/null +++ b/docs/cadence.md @@ -0,0 +1,48 @@ +# Benchmark schedules + +The schedule migration is incomplete. Historical repositories still own the +active weekly, canary and upstream-sync schedules. Do not enable duplicate +central schedules before removing the corresponding historical triggers. + +`suites/scheduled.json` selects the maintained suite and the new Moon, Pants, +Buck2, sbt and Nix workloads. It excludes Docker corpus and prospect drafts. +The same selections feed fresh dispatch and source checks. Case definitions own +workflow paths and variant inputs; the suite does not copy them. + +The proposed central cadence is: + +| Run | UTC schedule | CLI | Source | +| --- | --- | --- | --- | +| Weekly fresh | Monday 04:00 | Latest published stable release, with all required assets | Declared pins | +| Nightly fresh | Daily 01:17 | Latest published canary, with all required assets | Declared pins | +| Source check | Daily 00:11 | No build | Check upstream against declared pins | + +Fresh dispatch resolves one exact release tag for every selected workflow. It +validates all targets before dispatch and retains each requested run ID. A +dispatch succeeding does not establish that its builds passed. A dry run starts +no builds. Inspect the retained receipt before retrying an interrupted dispatch. + +Source checks currently retain proposals only. An unchanged source is recorded +as unchanged; a recipe mismatch or fixed source is recorded as blocked. A later +failure does not remove earlier case checks. The workflow does not commit source +changes, promote candidates or start rolling builds. + +## Work remaining before schedule migration + +- Connect reviewed source advancement to rolling dispatch and retain the exact + parent source, cache seed and requested run. Skip unchanged sources. Preserve + failed observations and serialize updates to each rolling cache. +- Add and qualify rolling paths for Immich, Mastodon, PostHog, n8n and OBS. Their + existing diagnostic paths do not establish the current comparison contract. +- Add and qualify source advancement and rolling paths for the seven new REAPI + and Nix cases. Their successful cold/warm screens use fixed source pins. +- Preserve Zed's build-verified source promotion and Deno's adjacent-commit + sequence when moving their current source controllers. +- Run the central dispatch on GitHub, check receipts and workload completion, + and verify the selected stable release supports each scheduled case. +- Remove replaced historical cron triggers, preserving manual and release + entrypoints. Change monitoring and publication ownership with that migration. + +The new central cron definitions are proposed configuration until this branch +is merged. Passing configuration tests does not complete the rolling or source +promotion work above. diff --git a/docs/process.md b/docs/process.md index 5dd65cec6..265662824 100644 --- a/docs/process.md +++ b/docs/process.md @@ -81,6 +81,9 @@ only when the experiment requires a different shape, and register it in the case Use the canonical Ruby reporter for every execution path. Add suite membership for an explicit scheduling purpose. `suites/published.json` is the shared registry for published reporting. +`suites/scheduled.json` selects the maintained suite and new native families for +the proposed shared cadence. See [benchmark schedules](cadence.md) for current +ownership and the remaining source and rolling work. `.github/actions/boringcache` is the sole BoringCache One invocation and release pin. It forwards the product inputs used by the cases and retains raw evidence. diff --git a/scripts/benchmark-cadence.rb b/scripts/benchmark-cadence.rb new file mode 100644 index 000000000..a344c5996 --- /dev/null +++ b/scripts/benchmark-cadence.rb @@ -0,0 +1,71 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require_relative "benchmark-cases" +require_relative "nightly-canaries" + +module BenchmarkCadence + class Error < StandardError; end + + def self.cases(root: BenchmarkCases::ROOT) + suite = JSON.parse(File.read(File.join(root, "suites/scheduled.json"))) + raise Error, "Unsupported scheduled suite version" unless suite.fetch("schema_version") == 1 + entries = suite.fetch("cases") + raise Error, "Scheduled cases must be unique" unless entries.uniq == entries + entries.each do |entry| + raise Error, "Use case_id and an optional variant" unless (entry.keys - %w[case_id variant]).empty? + BenchmarkCases.load_case(entry.fetch("case_id"), root) + end + entries + end + + def self.fresh_targets(root: BenchmarkCases::ROOT, case_id: nil) + cases(root: root).select { |entry| !case_id || entry.fetch("case_id") == case_id }.flat_map do |entry| + item = BenchmarkCases.load_case(entry.fetch("case_id"), root) + workflows = item.fetch("execution").fetch("workflows").select { |workflow| workflow.fetch("lane") == "fresh" } + raise Error, "#{item.fetch('id')} has no fresh workflow" if workflows.empty? + workflows.map do |workflow| + plan = BenchmarkCases.plan(item, lane: "fresh", workflow: File.basename(workflow.fetch("path")), variant: entry["variant"], root: root) + {"source_repo" => BenchmarkCases::REPOSITORY, "case_id" => item.fetch("id"), + "fresh_workflow" => plan.fetch("workflow"), "fresh_inputs" => plan.fetch("inputs")} + end + end + end + + def self.source_cases(root: BenchmarkCases::ROOT, case_id: nil) + ids = cases(root: root).map { |entry| entry.fetch("case_id") }.uniq + raise Error, "#{case_id} is not in the scheduled suite" if case_id && !ids.include?(case_id) + ids.select { |id| !case_id || id == case_id }.map { |id| BenchmarkCases.load_case(id, root) } + end +end + +if $PROGRAM_NAME == __FILE__ + options = {ref: "main"} + OptionParser.new do |parser| + parser.on("--check") { options[:check] = true } + parser.on("--collect") { options[:collect] = true } + parser.on("--case ID") { |value| options[:case_id] = value } + parser.on("--channel CHANNEL") { |value| options[:channel] = value } + parser.on("--version TAG") { |value| options[:version] = value } + parser.on("--ref REF") { |value| options[:ref] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--summary PATH") { |value| options[:summary] = value } + parser.on("--dry-run") { options[:dry_run] = true } + end.parse! + begin + targets = BenchmarkCadence.fresh_targets(case_id: options.delete(:case_id)) + raise BenchmarkCadence::Error, "No scheduled cases selected" if targets.empty? + if options.delete(:collect) + exit(NightlyCanaries::Runner.new.collect(summary: options.fetch(:summary), benchmarks: targets) ? 0 : 1) + elsif options.delete(:check) + puts "Validated #{targets.length} scheduled fresh workflows" + else + raise BenchmarkCadence::Error, "Use stable or canary" unless %w[stable canary].include?(options[:channel]) + raise BenchmarkCadence::Error, "Inspect the previous receipt before retrying a dispatch" if ENV.fetch("GITHUB_RUN_ATTEMPT", "1").to_i > 1 && !options[:dry_run] + NightlyCanaries::Runner.new.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, **options) + end + rescue BenchmarkCadence::Error, BenchmarkCases::Error, NightlyCanaries::Error, KeyError => error + abort error.message + end +end diff --git a/scripts/check-registry-alignment.rb b/scripts/check-registry-alignment.rb index bfda7a68e..c4beb9bdd 100755 --- a/scripts/check-registry-alignment.rb +++ b/scripts/check-registry-alignment.rb @@ -11,7 +11,8 @@ errors = [] suites.each do |name, suite| next unless suite.is_a?(Hash) - Array(suite["cases"]).each do |id| + Array(suite["cases"]).each do |entry| + id = entry.is_a?(Hash) ? entry.fetch("case_id") : entry item = by_id[id] errors << "#{name}: unknown case #{id}" unless item errors << "#{name}: retained case #{id} cannot execute" if item && item["kind"] == "retained" diff --git a/scripts/nightly-canaries.rb b/scripts/nightly-canaries.rb index 3cd2f8025..8d8060fb3 100755 --- a/scripts/nightly-canaries.rb +++ b/scripts/nightly-canaries.rb @@ -30,9 +30,24 @@ def published_canary(release) unless release["prerelease"] && !release["draft"] && release.fetch("tag_name").match?(/\Avcli-canary-[0-9a-f]{9,40}\z/) raise Error, "Use a published CLI canary" end + published_assets(release) + end + + def latest_stable + published_stable(api("repos/boringcache/cli/releases/latest")) + end + + def published_stable(release) + unless !release["prerelease"] && !release["draft"] && release.fetch("tag_name").match?(/\Av\d+\.\d+\.\d+\z/) + raise Error, "Use a published stable CLI release" + end + published_assets(release) + end + + def published_assets(release) names = release.fetch("assets").map { |asset| asset.fetch("name") } required = %w[SHA256SUMS boringcache-linux-amd64 boringcache-linux-arm64 boringcache-macos-universal boringcache-windows-amd64.exe] - raise Error, "The CLI canary is missing release assets" unless (required - names).empty? + raise Error, "The CLI release is missing release assets" unless (required - names).empty? release.fetch("tag_name") end @@ -52,17 +67,20 @@ def targets(benchmarks) end.uniq end - def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS) + def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS, channel: "canary", ref: "main") + raise Error, "Use stable or canary" unless %w[stable canary].include?(channel) selected = targets(benchmarks).select { |repo, _| repo == repository } raise Error, "No active fresh workflows registered for #{repository}" if selected.empty? if version.nil? || version.empty? - version = latest_canary + version = channel == "canary" ? latest_canary : latest_stable else - raise Error, "Use an exact CLI canary tag" unless version.match?(/\Avcli-canary-[0-9a-f]{9,40}\z/) - published_canary(api("repos/boringcache/cli/releases/tags/#{version}")) + pattern = channel == "canary" ? /\Avcli-canary-[0-9a-f]{9,40}\z/ : /\Av\d+\.\d+\.\d+\z/ + raise Error, "Use an exact CLI #{channel} tag" unless version.match?(pattern) + release = api("repos/boringcache/cli/releases/tags/#{version}") + channel == "canary" ? published_canary(release) : published_stable(release) end record = {"cli_version" => version, "created_at" => Time.now.utc.iso8601, - "state" => "dispatching", "runs" => []} + "channel" => channel, "ref" => ref, "state" => "dispatching", "runs" => []} selected.each do |repository, workflow, inputs| raise Error, "Benchmark lacks a fresh workflow" if repository.nil? || workflow.nil? record["runs"] << {"repository" => repository, "workflow" => workflow, "inputs" => inputs, "state" => "planned"} @@ -76,7 +94,7 @@ def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, b record["runs"].each do |run| next if dry_run result = api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}/dispatches", - body: {"ref" => "main", "inputs" => run.fetch("inputs").merge("cli_version" => version), "return_run_details" => true}) + body: {"ref" => ref, "inputs" => run.fetch("inputs").merge("cli_version" => version), "return_run_details" => true}) id = result["workflow_run_id"] raise Error, "Dispatch returned no run ID; inspect the workflow before retrying" unless id.is_a?(Integer) && id.positive? run.merge!("id" => id, "url" => "https://github.com/#{run.fetch('repository')}/actions/runs/#{id}", "state" => "requested") @@ -111,10 +129,11 @@ def check(record, summary:) def receipt(repository, run_id) Dir.mktmpdir("canary-receipt-") do |directory| + artifact = repository == "boringcache/benchmarks" ? "benchmark-dispatch" : "nightly-canaries" _, error, status = Open3.capture3("gh", "run", "download", run_id.to_s, - "--repo", repository, "--name", "nightly-canaries", "--dir", directory) + "--repo", repository, "--name", artifact, "--dir", directory) raise Error, "Cannot read dispatch receipt: #{error.strip}" unless status.success? - JSON.parse(File.read(File.join(directory, "nightly-canaries.json"))) + JSON.parse(File.read(File.join(directory, "#{artifact}.json"))) end end @@ -184,7 +203,7 @@ def write(path, record) def write_summary(path, record) return unless path - lines = ["## CLI canary benchmarks", "", "CLI: `#{record.fetch('cli_version')}`", "", + lines = ["## Fresh benchmark runs", "", "CLI: `#{record.fetch('cli_version')}`", "", "Dispatch: **#{record.fetch('state')}**. Benchmark outcomes are listed separately.", ""] record.fetch("runs").each do |run| name = "#{run.fetch('repository')} / #{workload_name(run)}" diff --git a/scripts/sync-sources.rb b/scripts/sync-sources.rb index 2521e80cc..83a23e96d 100755 --- a/scripts/sync-sources.rb +++ b/scripts/sync-sources.rb @@ -4,23 +4,40 @@ require "bundler/setup" require "optparse" require_relative "benchmark-cases" +require_relative "benchmark-cadence" -options = {} -OptionParser.new do |parser| - parser.on("--case ID") { |value| options[:case] = value } - parser.on("--output PATH") { |value| options[:output] = value } -end.parse! -raise "Use --output for the proposal inventory" unless options[:output] -items = options[:case] ? [BenchmarkCases.load_case(options[:case])] : BenchmarkCases.documents -items = items.select { |item| item.dig("execution", "sync") != "fixed" && item.dig("execution", "sync") && item.dig("execution", "blockers").to_a.empty? } -records = items.map do |item| - begin - BenchmarkCases.sync_source(item).merge("state" => "proposal", "requires_case_qualification" => true) - rescue BenchmarkCases::Error => error - {"case_id" => item.fetch("id"), "state" => "blocked", "error" => error.message} +module SourceSync + def self.propose(items, output:, sync: BenchmarkCases.method(:sync_source)) + inventory = {"schema_version" => 1, "records" => [], "publication" => "requires-review"} + BenchmarkCases.write_json(output, inventory) + items.each do |item| + record = begin + blockers = item.dig("execution", "blockers").to_a + raise BenchmarkCases::Error, blockers.join("; ") unless blockers.empty? + proposal = sync.call(item) + proposal.merge("state" => proposal.fetch("updated") ? "proposal" : "unchanged", + "requires_case_qualification" => proposal.fetch("updated")) + rescue BenchmarkCases::Error => error + {"case_id" => item.fetch("id"), "state" => "blocked", "error" => error.message} + end + inventory.fetch("records") << record + BenchmarkCases.write_json(output, inventory) + puts "#{record.fetch('case_id')}: #{record.fetch('state')}" + $stdout.flush + end + inventory end end -BenchmarkCases.validate -BenchmarkCases.write_json(options.fetch(:output), {"schema_version" => 1, "records" => records, "publication" => "requires-review"}) -puts "Prepared #{records.length} source proposals; #{records.count { |record| record['state'] == 'blocked' }} blocked" -exit(records.any? { |record| record["state"] == "blocked" } ? 1 : 0) + +if $PROGRAM_NAME == __FILE__ + options = {} + OptionParser.new do |parser| + parser.on("--case ID") { |value| options[:case] = value } + parser.on("--output PATH") { |value| options[:output] = value } + end.parse! + abort "Use --output for the proposal inventory" unless options[:output] + items = BenchmarkCadence.source_cases(case_id: options[:case]) + inventory = SourceSync.propose(items, output: options.fetch(:output)) + BenchmarkCases.validate + exit(inventory.fetch("records").any? { |record| record["state"] == "blocked" } ? 1 : 0) +end diff --git a/suites/scheduled.json b/suites/scheduled.json new file mode 100644 index 000000000..fe19e1c67 --- /dev/null +++ b/suites/scheduled.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "cases": [ + { + "case_id": "hugo" + }, + { + "case_id": "hugo-go" + }, + { + "case_id": "immich" + }, + { + "case_id": "mastodon", + "variant": "server" + }, + { + "case_id": "mastodon", + "variant": "streaming" + }, + { + "case_id": "posthog", + "variant": "layers" + }, + { + "case_id": "storybook" + }, + { + "case_id": "opentelemetry-java" + }, + { + "case_id": "spring-ai" + }, + { + "case_id": "grpc" + }, + { + "case_id": "zed", + "variant": "combined" + }, + { + "case_id": "deno", + "variant": "cargo-product" + }, + { + "case_id": "duckgres" + }, + { + "case_id": "chroma" + }, + { + "case_id": "linkerd2" + }, + { + "case_id": "qdrant" + }, + { + "case_id": "n8n", + "variant": "turbo" + }, + { + "case_id": "n8n", + "variant": "docker" + }, + { + "case_id": "n8n", + "variant": "runners" + }, + { + "case_id": "n8n", + "variant": "distroless" + }, + { + "case_id": "gogs-moon" + }, + { + "case_id": "opencut-moon" + }, + { + "case_id": "stackstorm-pants" + }, + { + "case_id": "executorch-buck2" + }, + { + "case_id": "msgpack-sbt" + }, + { + "case_id": "helix-nix" + }, + { + "case_id": "zed-nix" + }, + { + "case_id": "obs-studio", + "variant": "ccache" + }, + { + "case_id": "obs-studio", + "variant": "xcode" + } + ] +} diff --git a/test/benchmark_cadence_test.rb b/test/benchmark_cadence_test.rb new file mode 100644 index 000000000..099670690 --- /dev/null +++ b/test/benchmark_cadence_test.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/benchmark-cadence" +require_relative "../scripts/sync-sources" + +class BenchmarkCadenceTest < Minitest::Test + class Runner < NightlyCanaries::Runner + attr_reader :dispatches + + def initialize + @dispatches = [] + end + + def api(path, body: nil) + if body + @dispatches << {"path" => path, "body" => body} + return {"workflow_run_id" => @dispatches.length} + end + return {"state" => "active"} if path.include?("/actions/workflows/") + stable = path.end_with?("/latest") + release = {"tag_name" => stable ? "v1.40.0" : "vcli-canary-0123456789ab", "draft" => false, + "prerelease" => !stable, "published_at" => "2026-10-05T00:00:00Z", + "assets" => %w[SHA256SUMS boringcache-linux-amd64 boringcache-linux-arm64 boringcache-macos-universal boringcache-windows-amd64.exe].map { |name| {"name" => name} }} + stable ? release : [release] + end + end + + def test_weekly_and_nightly_dispatch_the_same_workloads_with_exact_release_tags + targets = BenchmarkCadence.fresh_targets + runners = %w[stable canary].map do |channel| + runner = Runner.new + Dir.mktmpdir do |directory| + record = runner.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, + channel: channel, output: File.join(directory, "receipt.json")) + assert_equal "requested", record.fetch("state") + assert_equal targets.length, record.fetch("runs").length + end + runner + end + stable, canary = runners.map(&:dispatches) + assert_equal stable.map { |run| run.fetch("path") }, canary.map { |run| run.fetch("path") } + stable.zip(canary).each do |weekly, nightly| + assert_equal "v1.40.0", weekly.dig("body", "inputs", "cli_version") + assert_equal "vcli-canary-0123456789ab", nightly.dig("body", "inputs", "cli_version") + assert_equal weekly.dig("body", "inputs").except("cli_version"), nightly.dig("body", "inputs").except("cli_version") + end + end + + def test_scheduled_suite_includes_maintained_cases_and_the_new_families + scheduled = BenchmarkCadence.cases.map { |entry| entry.fetch("case_id") } + maintained = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "suites/release.json"))).fetch("cases") + assert_empty maintained - scheduled + assert_empty %w[gogs-moon opencut-moon stackstorm-pants executorch-buck2 msgpack-sbt helix-nix zed-nix] - scheduled + refute_includes scheduled, "pants-jvm" + refute scheduled.any? { |id| id.start_with?("docker-") } + end + + def test_shared_workflows_keep_case_and_variant_selectors + targets = BenchmarkCadence.fresh_targets + native = targets.select { |item| item.fetch("fresh_workflow") == "reapi-fresh-benchmark.yml" } + assert_equal %w[executorch-buck2 gogs-moon msgpack-sbt opencut-moon stackstorm-pants], native.map { |item| item.dig("fresh_inputs", "case_id") }.sort + n8n = targets.select { |item| item.fetch("case_id") == "n8n" } + assert_equal %w[distroless docker runners turbo], n8n.map { |item| item.dig("fresh_inputs", "variant") }.sort + obs = targets.select { |item| item.fetch("case_id") == "obs-studio" } + assert_equal 4, obs.length + assert_equal 4, NightlyCanaries::Runner.new.targets(obs).length + end + + def test_source_checks_use_the_same_suite_once_per_case + ids = BenchmarkCadence.cases.map { |entry| entry.fetch("case_id") }.uniq + assert_equal ids, BenchmarkCadence.source_cases.map { |item| item.fetch("id") } + assert_equal ["helix-nix"], BenchmarkCadence.source_cases(case_id: "helix-nix").map { |item| item.fetch("id") } + assert_raises(BenchmarkCadence::Error) { BenchmarkCadence.source_cases(case_id: "pants-jvm") } + end + + def test_source_inventory_retains_unchanged_changed_and_blocked_cases + items = %w[hugo-go gogs-moon helix-nix].map { |id| BenchmarkCases.load_case(id) } + sync = lambda do |item| + id = item.fetch("id") + raise BenchmarkCases::Error, "Source recipe changed" if id == "helix-nix" + {"case_id" => id, "updated" => id == "hugo-go"} + end + Dir.mktmpdir do |directory| + path = File.join(directory, "sources.json") + inventory = nil + capture_io { inventory = SourceSync.propose(items, output: path, sync: sync) } + assert_equal inventory, JSON.parse(File.read(path)) + assert_equal %w[proposal unchanged blocked], inventory.fetch("records").map { |record| record.fetch("state") } + assert_equal false, inventory.fetch("records")[1].fetch("requires_case_qualification") + assert_equal "Source recipe changed", inventory.fetch("records")[2].fetch("error") + end + end +end From 0af9354452bdad9fea73ea9b9667ebcef594a69a Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 13:38:09 +0100 Subject: [PATCH 19/22] Gate cadence activation and verify CLI capabilities --- .github/workflows/canary.yml | 1 + .github/workflows/nightly-canaries.yml | 9 +++- .github/workflows/source-sync.yml | 1 + .github/workflows/weekly-fresh.yml | 1 + cases/chroma/case.json | 2 +- cases/duckgres/case.json | 2 +- cases/executorch-buck2/case.json | 2 +- cases/gogs-moon/case.json | 2 +- cases/helix-nix/case.json | 2 +- cases/hugo/case.json | 2 +- cases/immich/case.json | 11 ++++- cases/linkerd2/case.json | 2 +- cases/mastodon/case.json | 16 ++++++- cases/msgpack-sbt/case.json | 2 +- cases/n8n/case.json | 23 +++++++-- cases/opencut-moon/case.json | 2 +- cases/posthog/case.json | 15 +++++- cases/stackstorm-pants/case.json | 2 +- cases/zed-nix/case.json | 2 +- docs/cadence.md | 48 +++++++++++++++---- migration/cache-capacity-2026-10-05.json | 30 ++++++++++++ scripts/benchmark-cadence.rb | 41 +++++++++++++++- scripts/benchmark-cases.rb | 38 ++++++++++++++- scripts/nightly-canaries.rb | 3 +- scripts/reapi-setup.rb | 4 +- test/benchmark_cadence_test.rb | 48 +++++++++++++++++++ test/benchmark_cases_test.rb | 60 ++++++++++++++++++++++++ test/native_case_test.rb | 7 ++- 28 files changed, 344 insertions(+), 34 deletions(-) create mode 100644 migration/cache-capacity-2026-10-05.json diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index f151af8a1..c127267f5 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -20,6 +20,7 @@ concurrency: cancel-in-progress: false jobs: dispatch: + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' uses: ./.github/workflows/fresh-cadence.yml with: channel: canary diff --git a/.github/workflows/nightly-canaries.yml b/.github/workflows/nightly-canaries.yml index 4d24f63a0..89cc14966 100644 --- a/.github/workflows/nightly-canaries.yml +++ b/.github/workflows/nightly-canaries.yml @@ -19,7 +19,14 @@ jobs: - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - run: bundle install --jobs 4 --retry 2 - name: Check repository canaries - run: bundle exec ruby scripts/benchmark-cadence.rb --collect --summary "${GITHUB_STEP_SUMMARY}" + env: + CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + if [[ "$CADENCE_ACTIVE" == true ]]; then + bundle exec ruby scripts/benchmark-cadence.rb --collect --summary "$GITHUB_STEP_SUMMARY" + else + bundle exec ruby scripts/nightly-canaries.rb --collect --summary "$GITHUB_STEP_SUMMARY" + fi 'on': schedule: - cron: 47 * * * * diff --git a/.github/workflows/source-sync.yml b/.github/workflows/source-sync.yml index 72fe62a05..981034874 100644 --- a/.github/workflows/source-sync.yml +++ b/.github/workflows/source-sync.yml @@ -16,6 +16,7 @@ concurrency: cancel-in-progress: false jobs: propose: + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' runs-on: ubuntu-24.04 timeout-minutes: 60 steps: diff --git a/.github/workflows/weekly-fresh.yml b/.github/workflows/weekly-fresh.yml index 29e7197d5..556ee8bcf 100644 --- a/.github/workflows/weekly-fresh.yml +++ b/.github/workflows/weekly-fresh.yml @@ -17,6 +17,7 @@ permissions: actions: write jobs: dispatch: + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' concurrency: group: benchmark-weekly-dispatch cancel-in-progress: false diff --git a/cases/chroma/case.json b/cases/chroma/case.json index eb47dab4d..0abc6c3d8 100644 --- a/cases/chroma/case.json +++ b/cases/chroma/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/chroma-docker-benchmark", "benchmark_id": "chroma", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/duckgres/case.json b/cases/duckgres/case.json index 0e46cc0a7..a6180fa11 100644 --- a/cases/duckgres/case.json +++ b/cases/duckgres/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/duckgres-docker-benchmark", "benchmark_id": "duckgres", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/executorch-buck2/case.json b/cases/executorch-buck2/case.json index 068382899..dd69ab280 100644 --- a/cases/executorch-buck2/case.json +++ b/cases/executorch-buck2/case.json @@ -22,7 +22,7 @@ "revision": "5e21c13cc9e34fa2922d5708ba5962e3e365afdb" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", diff --git a/cases/gogs-moon/case.json b/cases/gogs-moon/case.json index 8c113a24b..274eec4e1 100644 --- a/cases/gogs-moon/case.json +++ b/cases/gogs-moon/case.json @@ -21,7 +21,7 @@ "revision": "dbbd717e923694c36f41b0360515b4374bbe6ee0" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", diff --git a/cases/helix-nix/case.json b/cases/helix-nix/case.json index bb314e586..f23e2d32b 100644 --- a/cases/helix-nix/case.json +++ b/cases/helix-nix/case.json @@ -21,7 +21,7 @@ "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/nix-fresh-benchmark.yml", diff --git a/cases/hugo/case.json b/cases/hugo/case.json index 1c483b0f9..99b8ffe30 100644 --- a/cases/hugo/case.json +++ b/cases/hugo/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/hugo-docker-benchmark", "benchmark_id": "hugo", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/immich/case.json b/cases/immich/case.json index 057bc2dfa..9c07a745e 100644 --- a/cases/immich/case.json +++ b/cases/immich/case.json @@ -33,6 +33,13 @@ "path": ".github/workflows/immich-immich-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "immich" + } } ], "sync": "upstream-head", @@ -47,7 +54,9 @@ "fresh_inputs": { "load_image": "true" }, - "rolling_inputs": {} + "rolling_inputs": { + "load_image": "true" + } } }, "reporting": { diff --git a/cases/linkerd2/case.json b/cases/linkerd2/case.json index 66a8ff1f5..f18f2ad84 100644 --- a/cases/linkerd2/case.json +++ b/cases/linkerd2/case.json @@ -44,7 +44,7 @@ "action": ".github/actions/linkerd2-docker-benchmark", "benchmark_id": "linkerd2-v2", "rolling_inputs": { - "push_image": "true" + "load_image": "true" }, "environment": {}, "fresh_inputs": { diff --git a/cases/mastodon/case.json b/cases/mastodon/case.json index 08c376a67..e85f408c0 100644 --- a/cases/mastodon/case.json +++ b/cases/mastodon/case.json @@ -39,6 +39,19 @@ "path": ".github/workflows/mastodon-mastodon-docker-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "mastodon" + }, + "variants": [ + "server", + "server-sccache", + "streaming" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -55,7 +68,8 @@ "load_image": "true" }, "rolling_inputs": { - "workload": "server" + "workload": "server", + "load_image": "true" }, "variants": { "server": {}, diff --git a/cases/msgpack-sbt/case.json b/cases/msgpack-sbt/case.json index 30c742ae4..d6cb644c4 100644 --- a/cases/msgpack-sbt/case.json +++ b/cases/msgpack-sbt/case.json @@ -31,7 +31,7 @@ } } ], - "sync": "fixed", + "sync": "upstream-head", "blockers": [] }, "reporting": { diff --git a/cases/n8n/case.json b/cases/n8n/case.json index 2aad3cfd2..d03785284 100644 --- a/cases/n8n/case.json +++ b/cases/n8n/case.json @@ -45,6 +45,20 @@ "path": ".github/workflows/n8n-n8n-docker-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "n8n" + }, + "variants": [ + "turbo", + "docker", + "runners", + "distroless" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -74,7 +88,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/n8n/Dockerfile", - "image": "n8n-benchmark" + "image": "n8n-benchmark", + "load_image": "true" } }, "runners": { @@ -87,7 +102,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile", - "image": "n8n-runners-benchmark" + "image": "n8n-runners-benchmark", + "load_image": "true" } }, "distroless": { @@ -100,7 +116,8 @@ }, "rolling_inputs": { "dockerfile_path": "upstream/docker/images/runners/Dockerfile.distroless", - "image": "n8n-runners-distroless-benchmark" + "image": "n8n-runners-distroless-benchmark", + "load_image": "true" } } } diff --git a/cases/opencut-moon/case.json b/cases/opencut-moon/case.json index 8b2023d12..3f8cfa70e 100644 --- a/cases/opencut-moon/case.json +++ b/cases/opencut-moon/case.json @@ -21,7 +21,7 @@ "revision": "e668010778568641babef2cc40be4703ae6916d6" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", diff --git a/cases/posthog/case.json b/cases/posthog/case.json index bfddc7ca3..dc86a3a28 100644 --- a/cases/posthog/case.json +++ b/cases/posthog/case.json @@ -38,6 +38,18 @@ "path": ".github/workflows/posthog-posthog-benchmark.yml", "lane": "diagnostic", "inputs": {} + }, + { + "path": ".github/workflows/native-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "posthog" + }, + "variants": [ + "layers", + "combined" + ], + "variant_input": "variant" } ], "sync": "upstream-head", @@ -54,7 +66,8 @@ "load_image": "true" }, "rolling_inputs": { - "platform": "linux/amd64" + "platform": "linux/amd64", + "load_image": "true" }, "variants": { "layers": {}, diff --git a/cases/stackstorm-pants/case.json b/cases/stackstorm-pants/case.json index ff36584be..78e123d87 100644 --- a/cases/stackstorm-pants/case.json +++ b/cases/stackstorm-pants/case.json @@ -21,7 +21,7 @@ "revision": "9824de4dfd0c869869e310dee729308f398ad83a" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/reapi-fresh-benchmark.yml", diff --git a/cases/zed-nix/case.json b/cases/zed-nix/case.json index 409ba47d6..82135887d 100644 --- a/cases/zed-nix/case.json +++ b/cases/zed-nix/case.json @@ -21,7 +21,7 @@ "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" }, "execution": { - "sync": "fixed", + "sync": "upstream-head", "workflows": [ { "path": ".github/workflows/nix-fresh-benchmark.yml", diff --git a/docs/cadence.md b/docs/cadence.md index 8d5a1d89d..0aa032778 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -3,6 +3,10 @@ The schedule migration is incomplete. Historical repositories still own the active weekly, canary and upstream-sync schedules. Do not enable duplicate central schedules before removing the corresponding historical triggers. +Central scheduled jobs require the repository variable +`BENCHMARK_CADENCE_ACTIVE=true`. Manual checks remain available while it is +unset. Canary monitoring continues to inspect the historical repositories until +that same switch changes ownership. `suites/scheduled.json` selects the maintained suite and the new Moon, Pants, Buck2, sbt and Nix workloads. It excludes Docker corpus and prospect drafts. @@ -21,21 +25,45 @@ Fresh dispatch resolves one exact release tag for every selected workflow. It validates all targets before dispatch and retains each requested run ID. A dispatch succeeding does not establish that its builds passed. A dry run starts no builds. Inspect the retained receipt before retrying an interrupted dispatch. +When the selection includes REAPI cases, preflight downloads the exact CLI, +verifies its release checksum, and requires `cache-registry --reapi-port` in its +command interface before dispatching any target. This check also runs in dry runs. -Source checks currently retain proposals only. An unchanged source is recorded -as unchanged; a recipe mismatch or fixed source is recorded as blocked. A later +The October 5 capability check rejected stable `v1.33.0` before dispatch: it +does not expose `--reapi-port`. Canary `vcli-canary-7a5b27146ebe` passed the same +31-target check. Stable activation remains pending a compatible release; the +weekly caller must not silently substitute a canary or omit the new families. + +Source checks currently retain proposals only. The seven new snapshot cases now +check their declared upstream branches, require fast-forward ancestry, and verify +the reviewed recipe against an exact candidate commit. Nix proposals update the +matching source URI in the plan and command contract; they do not change recipe +digests or build flags. A recipe or patch failure leaves the declared pin unchanged. +An unchanged source is recorded as unchanged; a recipe mismatch or fixed source +is recorded as blocked. A later failure does not remove earlier case checks. The workflow does not commit source changes, promote candidates or start rolling builds. +Immich, Mastodon, PostHog and n8n now register the shared native rolling workflow. +Their Docker variants load and inspect output in both providers. The same output +contract now applies to Hugo, Chroma, Duckgres and Linkerd2 rolling definitions, +which previously selected image publication. These definitions need live rolling +qualification; passing schema and workflow checks does not establish seed reuse. + +The repository's Actions cache limit was increased from 10 GB to 200 GB on +October 5. Retention is still seven days. This provides more capacity, but does +not prove that an individual rolling seed was available or restored. + ## Work remaining before schedule migration - Connect reviewed source advancement to rolling dispatch and retain the exact parent source, cache seed and requested run. Skip unchanged sources. Preserve failed observations and serialize updates to each rolling cache. -- Add and qualify rolling paths for Immich, Mastodon, PostHog, n8n and OBS. Their - existing diagnostic paths do not establish the current comparison contract. -- Add and qualify source advancement and rolling paths for the seven new REAPI - and Nix cases. Their successful cold/warm screens use fixed source pins. +- Qualify the registered rolling paths for Immich, Mastodon, PostHog and n8n, + and add and qualify OBS rolling. Their existing diagnostic paths do not + establish the current comparison contract. +- Qualify source advancement and add rolling paths for the seven new REAPI and + Nix cases. Their successful cold/warm screens use fixed source pins. - Preserve Zed's build-verified source promotion and Deno's adjacent-commit sequence when moving their current source controllers. - Run the central dispatch on GitHub, check receipts and workload completion, @@ -43,6 +71,8 @@ changes, promote candidates or start rolling builds. - Remove replaced historical cron triggers, preserving manual and release entrypoints. Change monitoring and publication ownership with that migration. -The new central cron definitions are proposed configuration until this branch -is merged. Passing configuration tests does not complete the rolling or source -promotion work above. +After a compatible stable release and qualification, remove replaced historical +cron triggers, verify there are no outstanding duplicate dispatches, then set +`BENCHMARK_CADENCE_ACTIVE=true`. Merging this branch alone does not activate the +central jobs. Passing configuration tests does not complete the rolling or +source promotion work above. diff --git a/migration/cache-capacity-2026-10-05.json b/migration/cache-capacity-2026-10-05.json new file mode 100644 index 000000000..bca710ed1 --- /dev/null +++ b/migration/cache-capacity-2026-10-05.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "repository": "boringcache/benchmarks", + "scope": "Repository capacity after the approved limit increase. These observations do not establish per-phase seed availability or restoration.", + "observations": [ + { + "collected_at": "2026-10-05T12:37:50Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/storage-limit", + "response": { + "max_cache_size_gb": 200 + } + }, + { + "collected_at": "2026-10-05T12:37:51Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/retention-limit", + "response": { + "max_cache_retention_days": 7 + } + }, + { + "collected_at": "2026-10-05T12:37:51Z", + "endpoint": "repos/boringcache/benchmarks/actions/cache/usage", + "response": { + "full_name": "boringcache/benchmarks", + "active_caches_size_in_bytes": 10135681922, + "active_caches_count": 262 + } + } + ] +} diff --git a/scripts/benchmark-cadence.rb b/scripts/benchmark-cadence.rb index a344c5996..5d93f4f5f 100644 --- a/scripts/benchmark-cadence.rb +++ b/scripts/benchmark-cadence.rb @@ -38,6 +38,44 @@ def self.source_cases(root: BenchmarkCases::ROOT, case_id: nil) raise Error, "#{case_id} is not in the scheduled suite" if case_id && !ids.include?(case_id) ids.select { |id| !case_id || id == case_id }.map { |id| BenchmarkCases.load_case(id, root) } end + + def self.rolling_targets(root: BenchmarkCases::ROOT, case_id:) + source_cases(root: root, case_id: case_id) + item = BenchmarkCases.load_case(case_id, root) + workflows = item.fetch("execution").fetch("workflows").select { |workflow| workflow.fetch("lane") == "rolling" } + raise Error, "#{case_id} has no rolling workflow" if workflows.empty? + cases(root: root).select { |entry| entry.fetch("case_id") == case_id }.flat_map do |entry| + workflows.map do |workflow| + BenchmarkCases.plan(item, lane: "rolling", workflow: File.basename(workflow.fetch("path")), + variant: entry["variant"], root: root) + end + end.uniq + end + + def self.verify_cli(version, runs, probe: method(:cli_help)) + return unless runs.any? { |run| run.fetch("workflow").start_with?("reapi-") } + help = probe.call(version, "cache-registry") + unless help.match?(/--reapi-port\b/) + raise Error, "#{version} does not support cache-registry --reapi-port required by the REAPI cases; no builds dispatched" + end + end + + def self.cli_help(version, command) + require_relative "reapi-setup" + platform = RbConfig::CONFIG.fetch("host_os") + asset = if platform.include?("darwin") + "boringcache-macos-universal" + elsif platform.include?("linux") + "boringcache-linux-#{RbConfig::CONFIG.fetch('host_cpu').match?(/aarch64|arm64/) ? 'arm64' : 'amd64'}" + else + raise Error, "CLI capability inspection requires Linux or macOS" + end + Dir.mktmpdir("benchmark-cli-") do |directory| + binary = File.join(directory, "boringcache") + ReapiSetup.download("boringcache/cli", version, asset, "boringcache", destination: binary) + BenchmarkCases.command(binary, command, "--help") + end + end end if $PROGRAM_NAME == __FILE__ @@ -63,7 +101,8 @@ def self.source_cases(root: BenchmarkCases::ROOT, case_id: nil) else raise BenchmarkCadence::Error, "Use stable or canary" unless %w[stable canary].include?(options[:channel]) raise BenchmarkCadence::Error, "Inspect the previous receipt before retrying a dispatch" if ENV.fetch("GITHUB_RUN_ATTEMPT", "1").to_i > 1 && !options[:dry_run] - NightlyCanaries::Runner.new.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, **options) + NightlyCanaries::Runner.new.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: targets, + preflight: BenchmarkCadence.method(:verify_cli), **options) end rescue BenchmarkCadence::Error, BenchmarkCases::Error, NightlyCanaries::Error, KeyError => error abort error.message diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 098bfe22c..12f713357 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -273,6 +273,7 @@ def self.update_source(item, workload, root: ROOT) def self.sync_source(item, root: ROOT) kind = item.fetch("execution").fetch("sync", "fixed") raise Error, "#{item.fetch('id')} has fixed source pins" if kind == "fixed" + return sync_snapshot(item, root: root) if kind == "upstream-head" && item.dig("source", "revision") if kind == "upstream-head" pin = item.fetch("source").fetch("pins").find { |value| value["kind"] == "gitlink" } branch, _, status = Open3.capture3("git", "config", "-f", File.join(root, "cases", item.fetch("id"), "payload", ".gitmodules"), "--get", "submodule.#{pin.fetch('path')}.branch") @@ -295,7 +296,7 @@ def self.sync_source(item, root: ROOT) raise Error, "Upstream source is not a fast-forward from the declared pin" unless comparison["status"] == "ahead" && comparison.dig("merge_base_commit", "sha") == current_sha verify_recipe(workload) update_source(item, workload, root: root) - {"case_id" => item.fetch("id"), "updated" => true, "head_sha" => next_sha} + {"case_id" => item.fetch("id"), "updated" => true, "base_sha" => current_sha, "head_sha" => next_sha} else old_source = File.read(File.join(workload, "benchmark-source.env")) output_path = File.join(directory, "outputs") @@ -326,6 +327,41 @@ def self.sync_source(item, root: ROOT) end end + def self.sync_snapshot(item, root: ROOT) + repository = item.dig("source", "repository") + current = item.dig("source", "revision") + branch = item.dig("origin", "branch") + head = JSON.parse(command("gh", "api", "repos/#{repository}/commits/#{CGI.escape(branch)}")).fetch("sha") + return {"case_id" => item.fetch("id"), "updated" => false} if head == current + raise Error, "Upstream returned an invalid revision" unless head.match?(/\A[0-9a-f]{40}\z/) + comparison = JSON.parse(command("gh", "api", "repos/#{repository}/compare/#{current}...#{head}")) + unless comparison["status"] == "ahead" && comparison.dig("merge_base_commit", "sha") == current + raise Error, "Upstream source is not a fast-forward from the declared pin" + end + candidate = Marshal.load(Marshal.dump(item)) + candidate.fetch("source")["revision"] = head + candidate.dig("source", "pins").each do |pin| + raise Error, "Snapshot has independently pinned sources; review them before advancement" unless pin["revision"] == current + pin["revision"] = head + end + Dir.mktmpdir("benchmark-source-") do |directory| + workload = prepare(candidate, directory: File.join(directory, "workload"), root: root) + command(RbConfig.ruby, File.join(workload, "scripts/verify-upstream-recipe.rb"), workload, chdir: workload) + end + # Only the exact source URI changes; recipe digests and build flags stay reviewed. + payload = File.join(root, "cases", item.fetch("id"), "payload") + reference = "github:#{repository}/#{current}#default" + replacement = "github:#{repository}/#{head}#default" + %w[.boringcache.toml recipe-contract.json].each do |name| + path = File.join(payload, name) + next unless File.file?(path) + content = File.read(path) + File.write(path, content.gsub(reference, replacement)) if content.include?(reference) + end + write_json(File.join(root, "cases", item.fetch("id"), "case.json"), candidate) + {"case_id" => item.fetch("id"), "updated" => true, "base_sha" => current, "head_sha" => head} + end + def self.verify_recipe(workload) path = File.join(workload, "recipe-contract.json") raise Error, "Source advancement requires a reviewed recipe contract" unless File.file?(path) diff --git a/scripts/nightly-canaries.rb b/scripts/nightly-canaries.rb index 8d8060fb3..c58c54853 100755 --- a/scripts/nightly-canaries.rb +++ b/scripts/nightly-canaries.rb @@ -67,7 +67,7 @@ def targets(benchmarks) end.uniq end - def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS, channel: "canary", ref: "main") + def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, benchmarks: BENCHMARKS, channel: "canary", ref: "main", preflight: nil) raise Error, "Use stable or canary" unless %w[stable canary].include?(channel) selected = targets(benchmarks).select { |repo, _| repo == repository } raise Error, "No active fresh workflows registered for #{repository}" if selected.empty? @@ -91,6 +91,7 @@ def dispatch(repository:, output:, summary: nil, version: nil, dry_run: false, b workflow = api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}") raise Error, "#{run.fetch('repository')} workflow is not active" unless workflow["state"] == "active" end + preflight.call(version, record.fetch("runs")) if preflight record["runs"].each do |run| next if dry_run result = api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}/dispatches", diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 27393d5b1..09875ecf0 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -12,7 +12,7 @@ def self.run(*command) raise "Command failed: #{command.first}" unless system(*command) end - def self.download(repository, release, asset, binary, sha256: nil) + def self.download(repository, release, asset, binary, sha256: nil, destination: nil) Dir.mktmpdir("reapi-download-") do |directory| run("gh", "release", "download", release, "--repo", repository, "--pattern", asset, "--dir", directory) path = File.join(directory, asset) @@ -22,7 +22,7 @@ def self.download(repository, release, asset, binary, sha256: nil) sha256 = rows.find { |row| row[1] == asset }&.first end raise "Release checksum is missing or different: #{asset}" unless sha256&.match?(/\A[0-9a-f]{64}\z/) && Digest::SHA256.file(path).hexdigest == sha256 - destination = File.join(ENV.fetch("RUNNER_TEMP"), "reapi-bin", binary) + destination ||= File.join(ENV.fetch("RUNNER_TEMP"), "reapi-bin", binary) FileUtils.mkdir_p(File.dirname(destination)) if asset.end_with?(".zst") run("zstd", "-d", path, "-o", destination) diff --git a/test/benchmark_cadence_test.rb b/test/benchmark_cadence_test.rb index 099670690..a69ca8b01 100644 --- a/test/benchmark_cadence_test.rb +++ b/test/benchmark_cadence_test.rb @@ -91,4 +91,52 @@ def test_source_inventory_retains_unchanged_changed_and_blocked_cases assert_equal "Source recipe changed", inventory.fetch("records")[2].fetch("error") end end + + def test_native_rolling_paths_verify_outputs_for_every_scheduled_variant + %w[hugo chroma duckgres linkerd2 immich mastodon posthog n8n].each do |id| + targets = BenchmarkCadence.rolling_targets(case_id: id) + selections = BenchmarkCadence.cases.select { |entry| entry.fetch("case_id") == id } + assert_equal selections.length, targets.length, id + assert targets.all? { |target| target.fetch("workflow") == "native-rolling-benchmark.yml" } + assert_equal selections.map { |entry| entry["variant"] }, targets.map { |target| target.dig("inputs", "variant") } + end + assert_raises(BenchmarkCadence::Error) { BenchmarkCadence.rolling_targets(case_id: "helix-nix") } + end + + def test_schedules_require_explicit_cutover_and_preserve_historical_monitoring + %w[weekly-fresh canary source-sync].each do |name| + workflow = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, ".github/workflows/#{name}.yml"))) + job = workflow.fetch("jobs").values.first + assert_equal "github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true'", job.fetch("if") + end + workflow = YAML.safe_load(File.read(File.join(BenchmarkCases::ROOT, ".github/workflows/nightly-canaries.yml"))) + step = workflow.dig("jobs", "results", "steps").find { |item| item["name"] == "Check repository canaries" } + assert_includes step.fetch("run"), 'scripts/nightly-canaries.rb --collect' + assert_includes step.fetch("run"), 'scripts/benchmark-cadence.rb --collect' + end + + def test_incompatible_cli_stops_the_entire_dispatch_and_retains_a_failed_receipt + runner = Runner.new + preflight = ->(version, runs) { BenchmarkCadence.verify_cli(version, runs, probe: ->(*) { "--port PORT" }) } + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + error = assert_raises(BenchmarkCadence::Error) do + runner.dispatch(repository: BenchmarkCases::REPOSITORY, benchmarks: BenchmarkCadence.fresh_targets, + channel: "stable", output: path, preflight: preflight) + end + assert_includes error.message, "--reapi-port" + assert_empty runner.dispatches + receipt = JSON.parse(File.read(path)) + assert_equal "dispatch-failed", receipt.fetch("state") + assert_equal 31, receipt.fetch("runs").length + assert receipt.fetch("runs").all? { |run| run.fetch("state") == "planned" } + end + end + + def test_cli_capability_probe_is_required_for_reapi_but_not_other_targets + runs = [{"workflow" => "reapi-fresh-benchmark.yml"}] + assert_nil BenchmarkCadence.verify_cli("v1.34.0", runs, probe: ->(*) { "--reapi-port PORT" }) + assert_nil BenchmarkCadence.verify_cli("v1.33.0", [{"workflow" => "native-fresh-benchmark.yml"}], + probe: ->(*) { flunk "No REAPI capability is needed" }) + end end diff --git a/test/benchmark_cases_test.rb b/test/benchmark_cases_test.rb index 6550f63aa..aa3cadcc2 100644 --- a/test/benchmark_cases_test.rb +++ b/test/benchmark_cases_test.rb @@ -168,6 +168,66 @@ def test_source_advancement_preserves_other_reviewed_source_pairs end end + def test_snapshot_advancement_verifies_before_changing_pins_or_nix_commands + Dir.mktmpdir do |root| + FileUtils.mkdir_p(File.join(root, "cases")) + FileUtils.cp_r(File.join(BenchmarkCases::ROOT, "cases/helix-nix"), File.join(root, "cases")) + item = BenchmarkCases.load_case("helix-nix", root) + base = item.dig("source", "revision") + head = "a" * 40 + calls = [] + command = lambda do |*args, **| + calls << args + if args.first == "gh" + args.last.include?("/compare/") ? JSON.generate({"status" => "ahead", "merge_base_commit" => {"sha" => base}}) : JSON.generate({"sha" => head}) + else + assert_equal base, BenchmarkCases.load_case("helix-nix", root).dig("source", "revision") + "Verified" + end + end + prepare = lambda do |candidate, **| + assert_equal head, candidate.dig("source", "revision") + assert_equal [head], candidate.dig("source", "pins").map { |pin| pin.fetch("revision") } + root + end + BenchmarkCases.stub(:command, command) do + BenchmarkCases.stub(:prepare, prepare) do + result = BenchmarkCases.sync_source(item, root: root) + assert_equal base, result.fetch("base_sha") + assert_equal head, result.fetch("head_sha") + end + end + assert_equal head, BenchmarkCases.load_case("helix-nix", root).dig("source", "revision") + assert_equal base, item.dig("source", "revision"), "Inspection must not mutate the original declaration" + payload = File.join(root, "cases/helix-nix/payload") + assert_includes File.read(File.join(payload, ".boringcache.toml")), "/#{head}#default" + contract = JSON.parse(File.read(File.join(payload, "recipe-contract.json"))) + original = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "cases/helix-nix/payload/recipe-contract.json"))) + assert_equal original.fetch("upstream_files"), contract.fetch("upstream_files") + assert calls.any? { |args| args.include?(File.join(root, "scripts/verify-upstream-recipe.rb")) } + end + end + + def test_snapshot_recipe_failure_leaves_definition_and_payload_unchanged + Dir.mktmpdir do |root| + FileUtils.mkdir_p(File.join(root, "cases")) + FileUtils.cp_r(File.join(BenchmarkCases::ROOT, "cases/helix-nix"), File.join(root, "cases")) + item = BenchmarkCases.load_case("helix-nix", root) + files = Dir.glob(File.join(root, "cases/helix-nix/**/{*,.*}")).select { |path| File.file?(path) } + original = files.to_h { |path| [path, File.binread(path)] } + command = lambda do |*args, **| + raise BenchmarkCases::Error, "Recipe changed" unless args.first == "gh" + args.last.include?("/compare/") ? JSON.generate({"status" => "ahead", "merge_base_commit" => {"sha" => item.dig("source", "revision")}}) : JSON.generate({"sha" => "a" * 40}) + end + BenchmarkCases.stub(:command, command) do + BenchmarkCases.stub(:prepare, ->(*, **) { root }) do + assert_raises(BenchmarkCases::Error) { BenchmarkCases.sync_source(item, root: root) } + end + end + assert_equal original, files.to_h { |path| [path, File.binread(path)] } + end + end + def test_source_sync_fetches_the_declared_branch_from_a_single_branch_shallow_checkout with_root do |root| git = ->(*args, directory:) { BenchmarkCases.command("git", "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args, chdir: directory) } diff --git a/test/native_case_test.rb b/test/native_case_test.rb index 45c165932..c50818b9e 100644 --- a/test/native_case_test.rb +++ b/test/native_case_test.rb @@ -13,13 +13,14 @@ def with_payload(id) end end - def test_fresh_and_rolling_share_the_recipe_but_preserve_publication_behavior + def test_fresh_and_rolling_share_the_recipe_and_loaded_output with_payload("hugo") do |item, directory| fresh = NativeCase.write_action(item, directory: directory, lane: "fresh") rolling = NativeCase.write_action(item, directory: directory, lane: "rolling") assert_equal fresh.dig("runs", "steps", 0, "uses"), rolling.dig("runs", "steps", 0, "uses") assert_equal "true", fresh.dig("runs", "steps", 0, "with", "load_image") - assert_equal "true", rolling.dig("runs", "steps", 0, "with", "push_image") + assert_equal "true", rolling.dig("runs", "steps", 0, "with", "load_image") + refute rolling.dig("runs", "steps", 0, "with").key?("push_image") assert_equal "${{ inputs.cli_version }}", fresh.dig("runs", "steps", 0, "with", "cli_version") end end @@ -129,6 +130,8 @@ def test_native_planning_rejects_missing_output_verification_before_dispatch %w[chroma duckgres hugo linkerd2].each do |id| item = BenchmarkCases.load_case(id) assert_equal "native-fresh-benchmark.yml", BenchmarkCases.plan(item).fetch("workflow") + assert_equal "native-rolling-benchmark.yml", BenchmarkCases.plan(item, lane: "rolling").fetch("workflow") + item.fetch("execution").fetch("native").fetch("rolling_inputs").delete("load_image") error = assert_raises(NativeCase::Error) { BenchmarkCases.plan(item, lane: "rolling") } assert_includes error.message, "requires load_image=true" end From 3e38aadb6cb9d9593b5c3dffe8dbec40200f1107 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 14:12:48 +0100 Subject: [PATCH 20/22] Fan out hourly source checks and fresh benchmark dispatch --- .github/workflows/fresh-cadence.yml | 99 +++++++++++++++- .github/workflows/native-fresh-benchmark.yml | 2 +- .../workflows/native-rolling-benchmark.yml | 2 +- .github/workflows/nix-fresh-benchmark.yml | 2 +- .github/workflows/reapi-fresh-benchmark.yml | 4 +- .github/workflows/source-sync.yml | 46 ++++++-- docs/cadence.md | 8 +- scripts/benchmark-cases.rb | 7 +- scripts/cadence-dispatch.rb | 107 ++++++++++++++++++ scripts/sync-sources.rb | 7 +- test/benchmark_cadence_test.rb | 15 +++ test/cadence_dispatch_test.rb | 72 ++++++++++++ 12 files changed, 349 insertions(+), 22 deletions(-) create mode 100644 scripts/cadence-dispatch.rb create mode 100644 test/cadence_dispatch_test.rb diff --git a/.github/workflows/fresh-cadence.yml b/.github/workflows/fresh-cadence.yml index cf0be3fd9..5fa0dd2b3 100644 --- a/.github/workflows/fresh-cadence.yml +++ b/.github/workflows/fresh-cadence.yml @@ -13,28 +13,117 @@ on: default: false permissions: contents: read - actions: write + actions: read jobs: - dispatch: + plan: + name: Validate suite and CLI runs-on: ubuntu-24.04 timeout-minutes: 15 + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - run: bundle install --jobs 4 --retry 2 - - name: Dispatch the scheduled suite + - name: Validate every target before requesting builds env: GH_TOKEN: ${{ github.token }} CHANNEL: ${{ inputs.channel }} CLI_VERSION: ${{ inputs.cli_version }} + run: | + ruby scripts/benchmark-cadence.rb --dry-run \ + --channel "$CHANNEL" --version "$CLI_VERSION" --ref "$GITHUB_REF_NAME" \ + --output "$RUNNER_TEMP/benchmark-dispatch.json" \ + --summary "$GITHUB_STEP_SUMMARY" + - id: matrix + run: | + echo "matrix=$(ruby scripts/cadence-dispatch.rb --plan "$RUNNER_TEMP/benchmark-dispatch.json" --matrix)" >> "$GITHUB_OUTPUT" + - name: Retain validated plan + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-plan + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 + - name: Retain failed preflight receipt + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-dispatch + path: ${{ runner.temp }}/benchmark-dispatch.json + if-no-files-found: error + retention-days: 90 + dispatch: + name: '${{ matrix.label }} / ${{ inputs.channel }} fresh' + needs: plan + if: '!inputs.dry_run' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + actions: write + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: benchmark-plan + path: ${{ runner.temp }}/plan + - name: Request this benchmark + env: + GH_TOKEN: ${{ github.token }} + TARGET_INDEX: ${{ matrix.index }} + run: | + ruby scripts/cadence-dispatch.rb \ + --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" --index "$TARGET_INDEX" \ + --output "$RUNNER_TEMP/target-$TARGET_INDEX.json" + - name: Retain this request + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-target-${{ matrix.index }} + path: ${{ runner.temp }}/target-${{ matrix.index }}.json + if-no-files-found: error + retention-days: 90 + receipt: + name: Collect requested runs + needs: [plan, dispatch] + if: always() && needs.plan.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: benchmark-plan + path: ${{ runner.temp }}/plan + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + if: '!inputs.dry_run' + with: + pattern: benchmark-target-* + path: ${{ runner.temp }}/receipts + - name: Combine requests without losing failed or missing targets + if: always() + env: DRY_RUN: ${{ inputs.dry_run }} run: | args=() [[ "$DRY_RUN" != true ]] || args+=(--dry-run) - bundle exec ruby scripts/benchmark-cadence.rb \ - --channel "$CHANNEL" --version "$CLI_VERSION" --ref "$GITHUB_REF_NAME" \ + ruby scripts/cadence-dispatch.rb \ + --plan "$RUNNER_TEMP/plan/benchmark-dispatch.json" \ + --receipts "$RUNNER_TEMP/receipts" \ --output "$RUNNER_TEMP/benchmark-dispatch.json" \ --summary "$GITHUB_STEP_SUMMARY" "${args[@]}" - name: Retain dispatch receipt diff --git a/.github/workflows/native-fresh-benchmark.yml b/.github/workflows/native-fresh-benchmark.yml index 58e76bf91..0ba1421f2 100644 --- a/.github/workflows/native-fresh-benchmark.yml +++ b/.github/workflows/native-fresh-benchmark.yml @@ -1,6 +1,6 @@ --- name: Native comparison fresh -run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample +run-name: "${{ inputs.case_id }} ${{ inputs.variant }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}" permissions: contents: read diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 37056a8ae..441a9e320 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -1,6 +1,6 @@ --- name: Native comparison rolling -run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample +run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample }}" permissions: contents: read diff --git a/.github/workflows/nix-fresh-benchmark.yml b/.github/workflows/nix-fresh-benchmark.yml index fc1b57967..2c7377b69 100644 --- a/.github/workflows/nix-fresh-benchmark.yml +++ b/.github/workflows/nix-fresh-benchmark.yml @@ -1,5 +1,5 @@ name: Nix comparison fresh -run-name: '${{ inputs.case_id }} Nix ${{ inputs.series_id }} sample ${{ inputs.sample }}' +run-name: '${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}' on: workflow_dispatch: inputs: diff --git a/.github/workflows/reapi-fresh-benchmark.yml b/.github/workflows/reapi-fresh-benchmark.yml index ac54f6100..733cbdba2 100644 --- a/.github/workflows/reapi-fresh-benchmark.yml +++ b/.github/workflows/reapi-fresh-benchmark.yml @@ -1,5 +1,5 @@ -name: Native REAPI screening -run-name: '${{ inputs.case_id }} REAPI ${{ inputs.series_id }} sample ${{ inputs.sample }}' +name: REAPI comparison fresh +run-name: '${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}' on: workflow_dispatch: inputs: diff --git a/.github/workflows/source-sync.yml b/.github/workflows/source-sync.yml index 981034874..42a8bbc0e 100644 --- a/.github/workflows/source-sync.yml +++ b/.github/workflows/source-sync.yml @@ -1,7 +1,7 @@ name: Check upstream source updates on: schedule: - - cron: '11 0 * * *' + - cron: '11 * * * *' workflow_dispatch: inputs: case_id: @@ -11,14 +11,39 @@ on: permissions: contents: read actions: read -concurrency: - group: benchmark-source-proposals - cancel-in-progress: false jobs: - propose: + select: if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 10 + outputs: + matrix: ${{ steps.cases.outputs.matrix }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - id: cases + env: + CASE_ID: ${{ inputs.case_id }} + run: | + args=() + [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") + echo "matrix=$(ruby scripts/sync-sources.rb --matrix "${args[@]}")" >> "$GITHUB_OUTPUT" + propose: + name: '${{ matrix.case_id }} source check' + needs: select + runs-on: ubuntu-24.04 timeout-minutes: 60 + concurrency: + group: benchmark-source-${{ matrix.case_id }} + cancel-in-progress: false + queue: max + strategy: + fail-fast: false + max-parallel: 6 + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: @@ -28,7 +53,7 @@ jobs: - name: Prepare reviewed source proposals env: GH_TOKEN: ${{ github.token }} - CASE_ID: ${{ inputs.case_id }} + CASE_ID: ${{ matrix.case_id }} run: | args=() [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") @@ -37,10 +62,13 @@ jobs: if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: source-proposals + name: source-proposals-${{ matrix.case_id }} path: | ${{ runner.temp }}/source-proposals.json - cases/*/case.json - cases/*/payload/benchmark-source.env + cases/${{ matrix.case_id }}/case.json + cases/${{ matrix.case_id }}/payload/benchmark-source.env + cases/${{ matrix.case_id }}/payload/recipe-contract.json + cases/${{ matrix.case_id }}/payload/.boringcache.toml + include-hidden-files: true if-no-files-found: error retention-days: 90 diff --git a/docs/cadence.md b/docs/cadence.md index 0aa032778..eea4d5131 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -19,10 +19,13 @@ The proposed central cadence is: | --- | --- | --- | --- | | Weekly fresh | Monday 04:00 | Latest published stable release, with all required assets | Declared pins | | Nightly fresh | Daily 01:17 | Latest published canary, with all required assets | Declared pins | -| Source check | Daily 00:11 | No build | Check upstream against declared pins | +| Source check | Hourly, at :11 | No build | Check upstream against declared pins | Fresh dispatch resolves one exact release tag for every selected workflow. It validates all targets before dispatch and retains each requested run ID. A +matrix requests each target independently; a failed request does not prevent +other targets from being requested. Each target retains its own receipt, and +the combined receipt keeps failed, uncertain and missing requests visible. A dispatch succeeding does not establish that its builds passed. A dry run starts no builds. Inspect the retained receipt before retrying an interrupted dispatch. When the selection includes REAPI cases, preflight downloads the exact CLI, @@ -43,6 +46,9 @@ An unchanged source is recorded as unchanged; a recipe mismatch or fixed source is recorded as blocked. A later failure does not remove earlier case checks. The workflow does not commit source changes, promote candidates or start rolling builds. +Each case has an independent job and artifact. A failed or slow source check +does not prevent the other cases from being checked. Checks for the same case +are serialized. Immich, Mastodon, PostHog and n8n now register the shared native rolling workflow. Their Docker variants load and inspect output in both providers. The same output diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index 12f713357..a2bd76406 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -284,8 +284,13 @@ def self.sync_source(item, root: ROOT) Dir.mktmpdir("benchmark-source-") do |directory| workload = prepare(item, directory: File.join(directory, "workload"), root: root) if kind == "upstream-head" - command("git", "submodule", "update", "--init", "--checkout", "upstream", chdir: workload) source = File.join(workload, "upstream") + # Inspect only the candidate tree. Ancestry is checked against the exact + # declared pin through the compare endpoint below. + unless File.exist?(File.join(source, ".git")) + command("git", "init", source) + command("git", "remote", "add", "origin", "https://github.com/#{item.dig('source', 'repository')}.git", chdir: source) + end command("git", "fetch", "--depth", "1", "origin", "refs/heads/#{branch}", chdir: source) next_sha = command("git", "rev-parse", "FETCH_HEAD", chdir: source).strip current_sha = item.fetch("source").fetch("pins").find { |pin| pin["kind"] == "gitlink" }.fetch("revision") diff --git a/scripts/cadence-dispatch.rb b/scripts/cadence-dispatch.rb new file mode 100644 index 000000000..30422297d --- /dev/null +++ b/scripts/cadence-dispatch.rb @@ -0,0 +1,107 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require_relative "benchmark-cadence" + +# The parent validates the entire suite and resolves one release before the +# matrix starts. Each child records its request independently, including errors. +module CadenceDispatch + class Error < StandardError; end + + def self.matrix(plan) + raise Error, "Expected a validated dispatch plan" unless plan.fetch("state") == "planned" + {"include" => plan.fetch("runs").each_with_index.map do |run, index| + {"index" => index, "label" => NightlyCanaries::Runner.new.workload_name(run)} + end} + end + + def self.dispatch(plan, index:, output:, runner: NightlyCanaries::Runner.new) + raise Error, "Inspect the previous receipt before retrying a dispatch" if ENV.fetch("GITHUB_RUN_ATTEMPT", "1").to_i > 1 + matrix(plan) + raise Error, "Invalid dispatch target index" unless index.is_a?(Integer) && index >= 0 && index < plan.fetch("runs").length + run = plan.fetch("runs").fetch(index).dup + record = plan.except("runs").merge("runs" => [run], "target_index" => index, "state" => "dispatching") + # A lost HTTP response cannot establish whether GitHub accepted the request. + run["state"] = "requesting" + runner.write(output, record) + result = runner.api("repos/#{run.fetch('repository')}/actions/workflows/#{run.fetch('workflow')}/dispatches", + body: {"ref" => plan.fetch("ref"), "inputs" => run.fetch("inputs").merge("cli_version" => plan.fetch("cli_version")), "return_run_details" => true}) + id = result["workflow_run_id"] + raise Error, "Dispatch returned no run ID; inspect the retained request before retrying" unless id.is_a?(Integer) && id.positive? + run.merge!("id" => id, "url" => "https://github.com/#{run.fetch('repository')}/actions/runs/#{id}", "state" => "requested") + record["state"] = "requested" + runner.write(output, record) + record + rescue StandardError => error + if record + run["state"] = "request-unknown" unless run["id"] + run["error"] = error.message + record.merge!("state" => "dispatch-failed", "error" => error.message) + runner.write(output, record) + end + raise + end + + def self.combine(plan, receipts, dry_run: false) + matrix(plan) + result = Marshal.load(Marshal.dump(plan)) + seen = [] + receipts.each do |receipt| + index = receipt.fetch("target_index") + raise Error, "Duplicate or invalid dispatch receipt" unless index.is_a?(Integer) && index >= 0 && index < result.fetch("runs").length && !seen.include?(index) + seen << index + %w[cli_version channel ref].each do |key| + raise Error, "Dispatch receipt #{key} differs from the plan" unless receipt.fetch(key) == plan.fetch(key) + end + raise Error, "Expected one target per receipt" unless receipt.fetch("runs").length == 1 + run = receipt.fetch("runs").first + expected = plan.fetch("runs").fetch(index) + %w[repository workflow inputs].each do |key| + raise Error, "Dispatch receipt target differs from the plan" unless run.fetch(key) == expected.fetch(key) + end + if run["state"] == "requested" + raise Error, "Requested run lacks its GitHub ID" unless run["id"].is_a?(Integer) && run["id"].positive? + end + result.fetch("runs")[index] = run + end + result["state"] = if dry_run && receipts.empty? + "planned" + elsif result.fetch("runs").all? { |run| run["state"] == "requested" } + "requested" + else + "dispatch-failed" + end + result + end +end + +if $PROGRAM_NAME == __FILE__ + options = {} + OptionParser.new do |parser| + parser.on("--plan PATH") { |value| options[:plan] = value } + parser.on("--matrix") { options[:matrix] = true } + parser.on("--index N", Integer) { |value| options[:index] = value } + parser.on("--receipts PATH") { |value| options[:receipts] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--summary PATH") { |value| options[:summary] = value } + parser.on("--dry-run") { options[:dry_run] = true } + end.parse! + begin + plan = JSON.parse(File.read(options.fetch(:plan))) + if options[:matrix] + puts JSON.generate(CadenceDispatch.matrix(plan)) + elsif options.key?(:index) + CadenceDispatch.dispatch(plan, index: options.fetch(:index), output: options.fetch(:output)) + else + receipts = Dir.glob(File.join(options.fetch(:receipts), "**/target-*.json")).sort.map { |path| JSON.parse(File.read(path)) } + record = CadenceDispatch.combine(plan, receipts, dry_run: options[:dry_run]) + runner = NightlyCanaries::Runner.new + runner.write(options.fetch(:output), record) + runner.write_summary(options[:summary], record) + exit(record.fetch("state") == "dispatch-failed" ? 1 : 0) + end + rescue CadenceDispatch::Error, NightlyCanaries::Error, KeyError, JSON::ParserError => error + abort error.message + end +end diff --git a/scripts/sync-sources.rb b/scripts/sync-sources.rb index 83a23e96d..2345e1633 100755 --- a/scripts/sync-sources.rb +++ b/scripts/sync-sources.rb @@ -34,9 +34,14 @@ def self.propose(items, output:, sync: BenchmarkCases.method(:sync_source)) OptionParser.new do |parser| parser.on("--case ID") { |value| options[:case] = value } parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--matrix") { options[:matrix] = true } end.parse! - abort "Use --output for the proposal inventory" unless options[:output] items = BenchmarkCadence.source_cases(case_id: options[:case]) + if options[:matrix] + puts JSON.generate({"case_id" => items.map { |item| item.fetch("id") }}) + exit + end + abort "Use --output for the proposal inventory" unless options[:output] inventory = SourceSync.propose(items, output: options.fetch(:output)) BenchmarkCases.validate exit(inventory.fetch("records").any? { |record| record["state"] == "blocked" } ? 1 : 0) diff --git a/test/benchmark_cadence_test.rb b/test/benchmark_cadence_test.rb index a69ca8b01..5035b7fa6 100644 --- a/test/benchmark_cadence_test.rb +++ b/test/benchmark_cadence_test.rb @@ -74,6 +74,21 @@ def test_source_checks_use_the_same_suite_once_per_case assert_raises(BenchmarkCadence::Error) { BenchmarkCadence.source_cases(case_id: "pants-jvm") } end + def test_source_matrix_selects_each_case_once_and_rejects_unknown_cases + script = File.join(BenchmarkCases::ROOT, "scripts/sync-sources.rb") + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix") + assert status.success?, output + ids = JSON.parse(output).fetch("case_id") + assert_equal BenchmarkCadence.source_cases.map { |item| item.fetch("id") }, ids + assert_equal ids.uniq, ids + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix", "--case", "helix-nix") + assert status.success?, output + assert_equal({"case_id" => ["helix-nix"]}, JSON.parse(output)) + output, status = Open3.capture2e(RbConfig.ruby, script, "--matrix", "--case", "unknown") + refute status.success? + assert_includes output, "not in the scheduled suite" + end + def test_source_inventory_retains_unchanged_changed_and_blocked_cases items = %w[hugo-go gogs-moon helix-nix].map { |id| BenchmarkCases.load_case(id) } sync = lambda do |item| diff --git a/test/cadence_dispatch_test.rb b/test/cadence_dispatch_test.rb new file mode 100644 index 000000000..22c7b1f39 --- /dev/null +++ b/test/cadence_dispatch_test.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/cadence-dispatch" + +class CadenceDispatchTest < Minitest::Test + def plan + {"state" => "planned", "cli_version" => "vcli-canary-0123456789ab", "channel" => "canary", "ref" => "main", + "runs" => %w[immich posthog].map { |id| {"repository" => "boringcache/benchmarks", "workflow" => "native-fresh-benchmark.yml", "inputs" => {"case_id" => id}, "state" => "planned"} }} + end + + def dispatch(index, response: {"workflow_run_id" => 123}) + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + runner = NightlyCanaries::Runner.new + request = lambda do |endpoint, body:| + assert_equal "requesting", JSON.parse(File.read(path)).dig("runs", 0, "state") + assert_equal "repos/boringcache/benchmarks/actions/workflows/native-fresh-benchmark.yml/dispatches", endpoint + assert_equal plan.fetch("cli_version"), body.dig("inputs", "cli_version") + assert_equal plan.dig("runs", index, "inputs", "case_id"), body.dig("inputs", "case_id") + response + end + runner.stub(:api, request) do + begin + result = CadenceDispatch.dispatch(plan, index: index, output: path, runner: runner) + assert_equal result, JSON.parse(File.read(path)) + result + rescue CadenceDispatch::Error + JSON.parse(File.read(path)) + end + end + end + end + + def test_independent_dispatch_receipts_combine_in_declared_order + first, second = dispatch(0), dispatch(1, response: {"workflow_run_id" => 456}) + combined = CadenceDispatch.combine(plan, [second, first]) + assert_equal "requested", combined.fetch("state") + assert_equal [123, 456], combined.fetch("runs").map { |run| run.fetch("id") } + assert_equal "planned", plan.fetch("state") + assert_equal [0, 1], CadenceDispatch.matrix(plan).fetch("include").map { |row| row.fetch("index") } + end + + def test_ambiguous_request_and_missing_target_are_retained + unknown = dispatch(0, response: {}) + assert_equal "dispatch-failed", unknown.fetch("state") + assert_equal "request-unknown", unknown.dig("runs", 0, "state") + combined = CadenceDispatch.combine(plan, [unknown]) + assert_equal "dispatch-failed", combined.fetch("state") + assert_equal %w[request-unknown planned], combined.fetch("runs").map { |run| run.fetch("state") } + end + + def test_failed_request_does_not_remove_successful_sibling + failed = dispatch(0, response: {}) + success = dispatch(1) + combined = CadenceDispatch.combine(plan, [failed, success]) + assert_equal "dispatch-failed", combined.fetch("state") + assert_equal %w[request-unknown requested], combined.fetch("runs").map { |run| run.fetch("state") } + end + + def test_duplicate_mismatched_and_incomplete_receipts_cannot_claim_success + receipt = dispatch(0) + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [receipt, receipt]) } + wrong_version = receipt.merge("cli_version" => "v1.33.0") + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [wrong_version]) } + wrong_index = receipt.merge("target_index" => 1) + assert_raises(CadenceDispatch::Error) { CadenceDispatch.combine(plan, [wrong_index]) } + assert_equal "dispatch-failed", CadenceDispatch.combine(plan, [receipt]).fetch("state") + assert_equal "planned", CadenceDispatch.combine(plan, [], dry_run: true).fetch("state") + assert_raises(CadenceDispatch::Error) { CadenceDispatch.matrix(plan.merge("state" => "dispatch-failed")) } + end +end From a36ecc7823847d3b4c1bb9b480b8b157534ced91 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 14:55:39 +0100 Subject: [PATCH 21/22] Connect source promotion and rolling workflows for the scheduled suite --- .github/actions/nix-benchmark/action.yml | 40 +- .github/actions/nix-environment/action.yml | 19 + .github/actions/reapi-benchmark/action.yml | 36 +- .../workflows/native-rolling-benchmark.yml | 14 +- .github/workflows/nix-fresh-benchmark.yml | 302 ++++++---- .github/workflows/nix-rolling-benchmark.yml | 215 +++++++ .github/workflows/obs-rolling-benchmark.yml | 531 ++++++++++++++++++ .github/workflows/reapi-fresh-benchmark.yml | 194 ++++--- .github/workflows/reapi-rolling-benchmark.yml | 142 +++++ .github/workflows/source-case.yml | 121 ++++ .github/workflows/source-sync.yml | 90 ++- .../workflows/zed-zed-cargo-rolling-auto.yml | 2 +- cases/executorch-buck2/case.json | 11 +- cases/gogs-moon/case.json | 11 +- cases/gogs-moon/payload/recipe-contract.json | 12 +- cases/helix-nix/case.json | 11 +- cases/immich/payload/recipe-contract.json | 10 +- cases/msgpack-sbt/case.json | 11 +- .../msgpack-sbt/payload/recipe-contract.json | 13 +- cases/obs-studio/case.json | 30 +- .../payload/scripts/verify_ccache_evidence.rb | 13 +- .../payload/scripts/write_phase_result.rb | 14 +- cases/opencut-moon/case.json | 11 +- cases/qdrant/payload/recipe-contract.json | 8 +- cases/stackstorm-pants/case.json | 11 +- cases/zed-nix/case.json | 11 +- cases/zed/case.json | 1 + cases/zed/payload/recipe-contract.json | 8 +- docs/cadence.md | 177 +++--- docs/recipe-reviews-2026-10-05.md | 24 + .../helix-nix/cadence-fresh-01/series.json | 51 ++ results/zed-nix/cadence-fresh-01/series.json | 51 ++ .../completions/37296214039.json | 17 + .../dispatches/1-nix-fresh-benchmark.yml.json | 37 ++ .../zed-nix/canary-screening-02/report.json | 424 ++++++++++++++ results/zed-nix/canary-screening-02/report.md | 36 ++ .../runs/1-boringcache-cold.json | 187 ++++++ .../runs/1-cachix-cold.json | 145 +++++ schemas/case.schema.json | 3 +- scripts/benchmark-cadence.rb | 9 +- scripts/benchmark-cases.rb | 13 + scripts/benchmark-phase.rb | 5 + scripts/check-report-contract.rb | 14 +- scripts/nix-benchmark.rb | 52 +- scripts/reapi-registry.rb | 2 +- scripts/reapi-setup.rb | 23 +- scripts/source-promotion.rb | 291 ++++++++++ scripts/verify-upstream-recipe.rb | 12 +- test/benchmark_cadence_test.rb | 7 +- test/benchmark_case_scripts_test.rb | 35 ++ test/benchmark_phase_test.rb | 10 + test/nix_benchmark_test.rb | 38 ++ test/reapi_registry_test.rb | 10 + test/reapi_report_test.rb | 6 +- test/source_promotion_test.rb | 182 ++++++ 55 files changed, 3376 insertions(+), 377 deletions(-) create mode 100644 .github/actions/nix-environment/action.yml create mode 100644 .github/workflows/nix-rolling-benchmark.yml create mode 100644 .github/workflows/obs-rolling-benchmark.yml create mode 100644 .github/workflows/reapi-rolling-benchmark.yml create mode 100644 .github/workflows/source-case.yml create mode 100644 docs/recipe-reviews-2026-10-05.md create mode 100644 results/helix-nix/cadence-fresh-01/series.json create mode 100644 results/zed-nix/cadence-fresh-01/series.json create mode 100644 results/zed-nix/canary-screening-02/completions/37296214039.json create mode 100644 results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json create mode 100644 results/zed-nix/canary-screening-02/report.json create mode 100644 results/zed-nix/canary-screening-02/report.md create mode 100644 results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json create mode 100644 results/zed-nix/canary-screening-02/runs/1-cachix-cold.json create mode 100644 scripts/source-promotion.rb create mode 100644 test/source_promotion_test.rb diff --git a/.github/actions/nix-benchmark/action.yml b/.github/actions/nix-benchmark/action.yml index ed91c7855..874dafdb0 100644 --- a/.github/actions/nix-benchmark/action.yml +++ b/.github/actions/nix-benchmark/action.yml @@ -7,44 +7,43 @@ inputs: required: true phase: required: true + lane: + default: fresh cli-version: default: '' cachix-cache: default: '' cachix-token: default: '' + dependency-seed: + default: '' runs: using: composite steps: - - name: Verify the source recipe - shell: bash - run: ruby scripts/verify-upstream-recipe.rb . - - name: Install Nix - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 - with: - install_url: https://releases.nixos.org/nix/nix-2.35.2/install - enable_kvm: false - extra_nix_config: | - accept-flake-config = false - experimental-features = nix-command flakes + - uses: ./.github/actions/nix-environment - name: Prepare the dependency store shell: bash + env: + DEPENDENCY_SEED: ${{ inputs.dependency-seed }} run: | - test "$(nix --version)" = 'nix (Nix) 2.35.2' - ruby scripts/nix-benchmark.rb prepare + if [[ -n "$DEPENDENCY_SEED" ]]; then + ruby scripts/nix-benchmark.rb import-dependencies "$DEPENDENCY_SEED" + else + ruby scripts/nix-benchmark.rb prepare + fi - name: Scope the cache and start setup timing id: setup shell: bash env: BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: | scope="$(ruby scripts/benchmark-phase.rb scope)" ruby scripts/scope-case-cache.rb "$scope" echo "started_at=$(date +%s.%N)" >> "$GITHUB_OUTPUT" - name: Set up BoringCache id: publish - if: inputs.provider == 'boringcache' && inputs.phase == 'cold' + if: inputs.provider == 'boringcache' && inputs.phase != 'warm' uses: ./.github/actions/boringcache with: mode: nix @@ -104,17 +103,20 @@ runs: BENCHMARK_ID: ${{ inputs.case-id }} PROVIDER: ${{ inputs.provider }} PHASE: ${{ inputs.phase }} + CACHE_LANE: ${{ inputs.lane }} SETUP_SECONDS: ${{ steps.timing.outputs.seconds }} EVIDENCE_PATH: ${{ steps.publish.outputs.evidence-path || steps.restore.outputs.evidence-path }} run: | source_repository="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "repository")')" source_sha="$(ruby -rjson -e 'puts JSON.parse(File.read("benchmark-context.json")).dig("source", "revision")')" build_seconds="$(ruby -rjson -e 'puts JSON.parse(File.read("nix-evidence/timing.json")).fetch("build_seconds")')" + args=() + [[ "$PHASE" != warm ]] || args+=(--cache-hit true) + [[ "$PHASE" != cold ]] || args+=(--cache-hit false) ruby scripts/benchmark-report.rb phase --verified-output \ - --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane fresh --phase "$PHASE" --mode nix \ + --benchmark "$BENCHMARK_ID" --strategy "$PROVIDER" --lane "$CACHE_LANE" --phase "$PHASE" --mode nix \ --restore-or-setup-seconds "$SETUP_SECONDS" --build-seconds "$build_seconds" \ - --cache-hit "$([[ "$PHASE" == warm ]] && echo true || echo false)" \ - --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH" + --source-repository "$source_repository" --source-sha "$source_sha" --evidence "$EVIDENCE_PATH" "${args[@]}" - name: Retain Nix checks and logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f @@ -125,6 +127,6 @@ runs: - name: Retain phase measurements uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }} + name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }} path: benchmark-results/ if-no-files-found: error diff --git a/.github/actions/nix-environment/action.yml b/.github/actions/nix-environment/action.yml new file mode 100644 index 000000000..f062acb52 --- /dev/null +++ b/.github/actions/nix-environment/action.yml @@ -0,0 +1,19 @@ +name: Prepare Nix benchmark environment +description: Verifies the declared recipe and installs the pinned Nix version. +runs: + using: composite + steps: + - name: Verify the source recipe + shell: bash + run: ruby scripts/verify-upstream-recipe.rb . + - name: Install Nix + uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 + with: + install_url: https://releases.nixos.org/nix/nix-2.35.2/install + enable_kvm: false + extra_nix_config: | + accept-flake-config = false + experimental-features = nix-command flakes + - name: Verify the Nix version + shell: bash + run: test "$(nix --version)" = 'nix (Nix) 2.35.2' diff --git a/.github/actions/reapi-benchmark/action.yml b/.github/actions/reapi-benchmark/action.yml index 45ecf0b19..7af8bb666 100644 --- a/.github/actions/reapi-benchmark/action.yml +++ b/.github/actions/reapi-benchmark/action.yml @@ -9,6 +9,8 @@ inputs: required: true cli-version: required: true + lane: + default: fresh runs: using: composite steps: @@ -35,6 +37,22 @@ runs: with: name: reapi-store-${{ inputs.case-id }} path: reapi-store + - name: Restore the rolling comparator store + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + id: rolling-cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: reapi-store + key: reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: | + reapi-${{ inputs.case-id }}-${{ env.BENCHMARK_ROLLING_SCOPE || env.BENCHMARK_SERIES_ID || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}- + - name: Record the restored comparator seed + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + shell: bash + env: + BENCHMARK_ID: ${{ inputs.case-id }} + RESTORED_CACHE_KEY: ${{ steps.rolling-cache.outputs.cache-matched-key }} + run: ruby scripts/reapi-setup.rb rolling-seed - name: Download cold output checks if: inputs.phase == 'warm' uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 @@ -48,7 +66,7 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} BORINGCACHE_WORKSPACE: boringcache/benchmarks run: boringcache ci run --oidc-provider github-actions -- ruby scripts/reapi-client.rb build - name: Run bazel-remote @@ -58,7 +76,7 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: ruby scripts/reapi-client.rb build - name: Verify outputs and record measurements shell: bash @@ -66,10 +84,20 @@ runs: PHASE: ${{ inputs.phase }} PROVIDER: ${{ inputs.provider }} BENCHMARK_ID: ${{ inputs.case-id }} - CACHE_LANE: fresh + CACHE_LANE: ${{ inputs.lane }} run: | ruby scripts/reapi-client.rb verify ruby scripts/reapi-setup.rb report + - name: Record the verified comparator source + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + shell: bash + run: ruby scripts/reapi-setup.rb publish-seed + - name: Publish the verified rolling comparator store + if: inputs.provider == 'bazel-remote' && inputs.lane == 'rolling' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: reapi-store + key: ${{ steps.rolling-cache.outputs.cache-primary-key }} - name: Retain the comparator store if: inputs.provider == 'bazel-remote' && inputs.phase == 'cold' uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f @@ -88,6 +116,6 @@ runs: - name: Retain phase measurements uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-fresh-${{ inputs.phase }} + name: phase-${{ inputs.case-id }}-${{ inputs.provider }}-${{ inputs.lane }}-${{ inputs.phase }} path: benchmark-results/ if-no-files-found: error diff --git a/.github/workflows/native-rolling-benchmark.yml b/.github/workflows/native-rolling-benchmark.yml index 441a9e320..07eadad28 100644 --- a/.github/workflows/native-rolling-benchmark.yml +++ b/.github/workflows/native-rolling-benchmark.yml @@ -1,7 +1,7 @@ --- name: Native comparison rolling -run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample - }}" +run-name: "${{ inputs.case_id }} ${{ inputs.variant }} rolling ${{ inputs.series_id + }} sample ${{ inputs.sample }}" permissions: contents: read actions: read @@ -10,6 +10,7 @@ env: BORINGCACHE_NO_GIT: 'true' BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" jobs: publish: name: "${{ matrix.provider }} ${{ inputs.case_id }} commit" @@ -151,6 +152,9 @@ jobs: default: '' description: Reviewed native workload variant; required when the case declares variants. + cache_scope: + type: string + default: '' workflow_call: inputs: cli_version: @@ -188,7 +192,11 @@ jobs: type: string required: false default: '' + cache_scope: + type: string + default: '' concurrency: - group: benchmark-rolling-${{ inputs.case_id }}-${{ inputs.series_id || github.ref_name }} + group: benchmark-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} cancel-in-progress: false queue: max diff --git a/.github/workflows/nix-fresh-benchmark.yml b/.github/workflows/nix-fresh-benchmark.yml index 2c7377b69..0dacd26dc 100644 --- a/.github/workflows/nix-fresh-benchmark.yml +++ b/.github/workflows/nix-fresh-benchmark.yml @@ -1,37 +1,13 @@ +--- name: Nix comparison fresh -run-name: '${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}' -on: - workflow_dispatch: - inputs: - case_id: - type: choice - options: [helix-nix, zed-nix] - required: true - provider: - description: A single provider is a correctness screen, not a comparison. - type: choice - options: [all, boringcache, cachix] - default: all - cachix_cache: - description: Cachix cache name; empty uses the repository variable. - type: string - default: '' - cli_version: - description: Exact released CLI or canary; empty uses the Action default. - type: string - default: '' - series_id: - type: string - default: '' - sample: - type: string - default: '1' +run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample + }}" permissions: contents: read actions: read env: - BENCHMARK_SERIES_ID: ${{ inputs.series_id }} - BENCHMARK_SAMPLE: ${{ inputs.sample }} + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" BENCHMARK_RUNNER_CLASS: ubuntu-24.04 BORINGCACHE_NO_GIT: 'true' jobs: @@ -39,29 +15,66 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - matrix: ${{ steps.providers.outputs.matrix }} + matrix: "${{ steps.providers.outputs.matrix }}" steps: - - id: providers - env: - PROVIDER: ${{ inputs.provider }} - CACHIX_CACHE: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: | - if [[ "$PROVIDER" != boringcache ]]; then - test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } - test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } - fi - case "$PROVIDER" in - all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; - boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; - cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac - cold: - name: '${{ matrix.provider }} ${{ inputs.case_id }} cold' + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + CACHIX_CACHE: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + CACHIX_AUTH_TOKEN: "${{ secrets.CACHIX_AUTH_TOKEN }}" + run: | + if [[ "$PROVIDER" != boringcache ]]; then + test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } + test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } + fi + case "$PROVIDER" in + all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + dependencies: + name: "${{ inputs.case_id }} dependency seed" needs: providers runs-on: ubuntu-24.04 timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/nix-environment" + - name: Prepare one dependency store for every measured worker + run: | + ruby scripts/nix-benchmark.rb prepare + ruby scripts/nix-benchmark.rb export-dependencies + - name: Retain the common dependency store + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies/ + compression-level: 0 + if-no-files-found: error + retention-days: 7 + - name: Retain dependency preparation evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependency-evidence-${{ inputs.case_id }} + path: nix-evidence/ + if-no-files-found: error + retention-days: 90 + cold: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} cold" + needs: + - providers + - dependencies + runs-on: ubuntu-24.04 + timeout-minutes: 180 permissions: contents: read actions: read @@ -69,26 +82,37 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/nix-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: cold - cli-version: ${{ inputs.cli_version }} - cachix-cache: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - cachix-token: ${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN || '' }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: cold + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + dependency-seed: nix-dependencies warm: - name: '${{ matrix.provider }} ${{ inputs.case_id }} warm' - needs: [providers, cold] + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} warm" + needs: + - providers + - cold + - dependencies runs-on: ubuntu-24.04 timeout-minutes: 180 permissions: @@ -98,41 +122,125 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/nix-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: warm - cli-version: ${{ inputs.cli_version }} - cachix-cache: ${{ inputs.cachix_cache || vars.CACHIX_CACHE }} - cachix-token: ${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN || '' }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: warm + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + dependency-seed: nix-dependencies compare: if: inputs.provider == 'all' needs: warm runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - run: bundle install --jobs 4 --retry 2 - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 - with: - pattern: nix-*-cold - path: checks - - name: Compare provider outputs and dependency stores - env: - CASE_ID: ${{ inputs.case_id }} - run: | - bundle exec ruby -r ./scripts/nix-benchmark -e \ - 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-cold", current: "checks/nix-#{id}-cachix-cold")' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: nix-*-cold + path: checks + - name: Compare provider outputs and dependency stores + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + bundle exec ruby -r ./scripts/nix-benchmark -e \ + 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-cold", current: "checks/nix-#{id}-cachix-cold")' + report: + name: "${{ inputs.case_id }} fresh report" + needs: + - cold + - warm + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} fresh benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-fresh + path: benchmark-results/${{ inputs.case_id }}-boringcache-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain cachix result + if: inputs.provider == 'all' || inputs.provider == 'cachix' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-cachix-fresh + path: benchmark-results/${{ inputs.case_id }}-cachix-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-nix-fresh + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - helix-nix + - zed-nix + required: true + provider: + description: A single provider is a correctness screen, not a comparison. + type: choice + options: + - all + - boringcache + - cachix + default: all + cachix_cache: + description: Cachix cache name; empty uses the repository variable. + type: string + default: '' + cli_version: + description: Exact released CLI or canary; empty uses the Action default. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' diff --git a/.github/workflows/nix-rolling-benchmark.yml b/.github/workflows/nix-rolling-benchmark.yml new file mode 100644 index 000000000..59bce02d4 --- /dev/null +++ b/.github/workflows/nix-rolling-benchmark.yml @@ -0,0 +1,215 @@ +--- +name: Nix comparison rolling +run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample + }}" +permissions: + contents: read + actions: read +env: + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_RUNNER_CLASS: ubuntu-24.04 + BORINGCACHE_NO_GIT: 'true' + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" +jobs: + providers: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: "${{ steps.providers.outputs.matrix }}" + steps: + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + CACHIX_CACHE: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + CACHIX_AUTH_TOKEN: "${{ secrets.CACHIX_AUTH_TOKEN }}" + run: | + if [[ "$PROVIDER" != boringcache ]]; then + test -n "$CACHIX_CACHE" || { echo 'Set the CACHIX_CACHE repository variable'; exit 1; } + test -n "$CACHIX_AUTH_TOKEN" || { echo 'Set the CACHIX_AUTH_TOKEN repository secret'; exit 1; } + fi + case "$PROVIDER" in + all) echo 'matrix=["boringcache","cachix"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + cachix) echo 'matrix=["cachix"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + dependencies: + name: "${{ inputs.case_id }} dependency seed" + needs: providers + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/nix-environment" + - name: Prepare one dependency store for every measured worker + run: | + ruby scripts/nix-benchmark.rb prepare + ruby scripts/nix-benchmark.rb export-dependencies + - name: Retain the common dependency store + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies/ + compression-level: 0 + if-no-files-found: error + retention-days: 7 + - name: Retain dependency preparation evidence + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: nix-dependency-evidence-${{ inputs.case_id }} + path: nix-evidence/ + if-no-files-found: error + retention-days: 90 + commit: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} commit" + needs: + - providers + - dependencies + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + actions: read + id-token: write + strategy: + fail-fast: false + matrix: + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - name: Download the common dependency store + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: nix-dependencies-${{ inputs.case_id }} + path: nix-dependencies + - uses: "./.github/actions/nix-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: commit + cli-version: "${{ inputs.cli_version }}" + cachix-cache: "${{ inputs.cachix_cache || vars.CACHIX_CACHE }}" + cachix-token: "${{ matrix.provider == 'cachix' && secrets.CACHIX_AUTH_TOKEN + || '' }}" + lane: rolling + dependency-seed: nix-dependencies + compare: + if: inputs.provider == 'all' + needs: commit + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: nix-*-commit + path: checks + - name: Compare provider outputs and dependency stores + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + bundle exec ruby -r ./scripts/nix-benchmark -e \ + 'id = ENV.fetch("CASE_ID"); NixBenchmark.compare(seed: "checks/nix-#{id}-boringcache-commit", current: "checks/nix-#{id}-cachix-commit")' + report: + name: "${{ inputs.case_id }} rolling report" + needs: + - commit + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} rolling benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-rolling + path: benchmark-results/${{ inputs.case_id }}-boringcache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain cachix result + if: inputs.provider == 'all' || inputs.provider == 'cachix' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-cachix-rolling + path: benchmark-results/${{ inputs.case_id }}-cachix-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-nix-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +concurrency: + group: benchmark-nix-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} + cancel-in-progress: false + queue: max +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - helix-nix + - zed-nix + required: true + provider: + description: A single provider is a correctness screen, not a comparison. + type: choice + options: + - all + - boringcache + - cachix + default: all + cachix_cache: + description: Cachix cache name; empty uses the repository variable. + type: string + default: '' + cli_version: + description: Exact released CLI or canary; empty uses the Action default. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' diff --git a/.github/workflows/obs-rolling-benchmark.yml b/.github/workflows/obs-rolling-benchmark.yml new file mode 100644 index 000000000..cb84978e9 --- /dev/null +++ b/.github/workflows/obs-rolling-benchmark.yml @@ -0,0 +1,531 @@ +--- +name: OBS Studio comparison rolling +run-name: obs-studio ${{ inputs.cache_tool }} rolling ${{ inputs.series_id }} sample + ${{ inputs.sample }} +'on': + workflow_dispatch: + inputs: + cache_tool: + type: choice + options: + - ccache + - xcode + - all + default: all + cli_version: + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' + base_sha: + type: string + default: '' + head_sha: + type: string + default: '' + runner_label: + type: string + default: '' +permissions: + actions: read + contents: read +env: + BENCHMARK_ID: obs-studio-compiler-cache + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + CACHE_COHORT: "${{ inputs.cache_scope || inputs.series_id || github.ref_name }}" +concurrency: + group: benchmark-obs-rolling-${{ inputs.cache_scope || inputs.series_id || github.ref_name + }} + cancel-in-progress: false + queue: max +jobs: + actions-cache-ccache: + name: GitHub Actions obs-studio ccache commit + runs-on: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' + && inputs.runner_label || 'ubuntu-26.04' }}" + timeout-minutes: 120 + env: + CCACHE_DIR: "${{ github.workspace }}/.ccache" + CCACHE_BASEDIR: "${{ github.workspace }}/upstream" + CCACHE_NOHASHDIR: '1' + CCACHE_COMPILERCHECK: content + CCACHE_STATSLOG: "${{ github.workspace }}/.ccache-stats.log" + BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name + == 'main' && inputs.runner_label || 'ubuntu-26.04' }}" + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source + run: "./scripts/prepare-source.sh rolling" + - name: Install the upstream Linux build shell + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> "$GITHUB_PATH" + brew install --quiet zsh + - name: Install the benchmark ccache release + run: "./scripts/install-ccache.sh" + - name: Start cache restore timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore ccache directory + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: |- + .ccache + .rolling-seed.json + key: gha-obs-ccache-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: gha-obs-ccache-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ + runner.arch }}- + id: cache-restore + - name: Capture cache restore timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Retain the previous cache source + run: if [[ -f .rolling-seed.json ]]; then cp .rolling-seed.json benchmark-results/previous-cache-source.json; + fi + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/ccache-actions-cache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb ccache --working-directory upstream \ + 2>&1 | tee benchmark-results/ccache-actions-cache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/ccache-actions-cache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Capture and verify ccache evidence + run: | + ccache --print-log-stats --format=json > benchmark-results/ccache-actions-cache-rolling-native.json + ./scripts/verify_ccache_evidence.rb \ + --path benchmark-results/ccache-actions-cache-rolling-native.json \ + --phase commit \ + --strategy actions-cache + - name: Record the verified cache source + env: + SOURCE_SHA: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + run: 'ruby -rjson -e ''File.write(".rolling-seed.json", JSON.pretty_generate({source_sha: + ENV.fetch("SOURCE_SHA"), run_id: ENV.fetch("GITHUB_RUN_ID"), run_attempt: + ENV.fetch("GITHUB_RUN_ATTEMPT")}))''' + - name: Publish the verified ccache cache + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + key: "${{ steps.cache-restore.outputs.cache-primary-key }}" + path: |- + .ccache + .rolling-seed.json + - name: Write rolling result + env: + GITHUB_TOKEN: "${{ github.token }}" + RESTORED_CACHE_HIT: "${{ steps.cache-restore.outputs.cache-matched-key != + '' }}" + run: | + ./scripts/write_phase_result.rb \ + --surface ccache \ + --strategy actions-cache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --cache-key "${{ steps.cache-restore.outputs.cache-matched-key }}" \ + --native-evidence benchmark-results/ccache-actions-cache-rolling-native.json + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-actions-cache-ccache-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + if: inputs.cache_tool != 'xcode' && (inputs.runner_label == '' || github.ref_name + == 'main') + actions-cache-xcode: + name: GitHub Actions obs-studio xcode commit + runs-on: macos-26 + timeout-minutes: 120 + env: + XCODE_CAS_PATH: "${{ github.workspace }}/.xcode-compilation-cache" + OBS_BASELINE_DERIVED_DATA_PATH: "${{ github.workspace }}/.xcode-derived-data" + BENCHMARK_RUNNER_CLASS: macos-26 + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source + run: "./scripts/prepare-source.sh rolling" + - name: Start cache restore timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore Xcode compilation cache + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + path: |- + .xcode-compilation-cache + .rolling-seed.json + key: gha-obs-xcode-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: gha-obs-xcode-${{ env.CACHE_COHORT }}-${{ runner.os }}-${{ runner.arch + }}- + id: cache-restore + - name: Capture cache restore timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Retain the previous cache source + run: if [[ -f .rolling-seed.json ]]; then cp .rolling-seed.json benchmark-results/previous-cache-source.json; + fi + - name: Select upstream's pinned Xcode + run: sudo xcode-select --switch "${{ steps.source.outputs.OBS_XCODE_PATH }}" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/xcode-actions-cache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb xcode --working-directory upstream \ + 2>&1 | tee benchmark-results/xcode-actions-cache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/xcode-actions-cache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Require native Xcode build evidence + run: "./scripts/verify_xcode_build_log.rb benchmark-results/xcode-actions-cache-rolling.log" + - name: Record the verified cache source + env: + SOURCE_SHA: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + run: 'ruby -rjson -e ''File.write(".rolling-seed.json", JSON.pretty_generate({source_sha: + ENV.fetch("SOURCE_SHA"), run_id: ENV.fetch("GITHUB_RUN_ID"), run_attempt: + ENV.fetch("GITHUB_RUN_ATTEMPT")}))''' + - name: Publish the verified xcode cache + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae + with: + key: "${{ steps.cache-restore.outputs.cache-primary-key }}" + path: |- + .xcode-compilation-cache + .rolling-seed.json + - name: Write rolling result + env: + GITHUB_TOKEN: "${{ github.token }}" + RESTORED_CACHE_HIT: "${{ steps.cache-restore.outputs.cache-matched-key != + '' }}" + run: | + ./scripts/write_phase_result.rb \ + --surface xcode \ + --strategy actions-cache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --cache-key "${{ steps.cache-restore.outputs.cache-matched-key }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-actions-cache-xcode-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + if: inputs.cache_tool != 'ccache' && (inputs.runner_label == '' || github.ref_name + == 'main') + boringcache-ccache: + name: BoringCache obs-studio ccache commit + runs-on: "${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' + && inputs.runner_label || 'ubuntu-26.04' }}" + timeout-minutes: 120 + env: + CCACHE_DIR: "${{ github.workspace }}/.boringcache-local-ccache" + CCACHE_BASEDIR: "${{ github.workspace }}/upstream" + CCACHE_NOHASHDIR: '1' + CCACHE_COMPILERCHECK: content + BENCHMARK_RUNNER_CLASS: "${{ github.event_name == 'workflow_dispatch' && github.ref_name + == 'main' && inputs.runner_label || 'ubuntu-26.04' }}" + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source and matching cache plan + run: | + ./scripts/prepare-source.sh rolling + ruby scripts/scope-case-cache.rb "obs-studio-$CACHE_COHORT" + - name: Install the upstream Linux build shell + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> "$GITHUB_PATH" + brew install --quiet zsh + - name: Install the benchmark ccache release + run: "./scripts/install-ccache.sh" + - name: Start cache setup timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore and publish BoringCache ccache + id: cache + uses: "./.github/actions/boringcache" + with: + cli-version: "${{ inputs.cli_version }}" + trust-policy: publish + mode: ccache + working-directory: "./upstream" + - name: Capture cache setup timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/ccache-boringcache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb ccache --working-directory upstream \ + 2>&1 | tee benchmark-results/ccache-boringcache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/ccache-boringcache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Capture and verify native remote evidence + run: | + ccache --print-log-stats --format=json > benchmark-results/ccache-boringcache-rolling-native.json + ./scripts/verify_ccache_evidence.rb \ + --path benchmark-results/ccache-boringcache-rolling-native.json \ + --phase commit \ + --strategy boringcache + - name: Write rolling result + run: | + ./scripts/write_phase_result.rb \ + --surface ccache \ + --strategy boringcache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --action-evidence "${{ steps.cache.outputs.evidence-path }}" \ + --native-evidence benchmark-results/ccache-boringcache-rolling-native.json + env: + RESTORED_CACHE_HIT: "${{ steps.cache.outputs.cache-hit }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-boringcache-ccache-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + - name: Retain BoringCache product evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: boringcache-product-ccache-rolling + path: "${{ steps.cache.outputs.evidence-path }}" + if-no-files-found: error + retention-days: 30 + if: always() + permissions: + actions: read + contents: read + id-token: write + if: inputs.cache_tool != 'xcode' && (inputs.runner_label == '' || github.ref_name + == 'main') + boringcache-xcode: + name: BoringCache obs-studio xcode commit + if: inputs.cache_tool != 'ccache' && (inputs.runner_label == '' || github.ref_name + == 'main') + runs-on: macos-26 + timeout-minutes: 120 + steps: + - name: Checkout benchmark definitions + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - name: Prepare declared case + uses: "./.harness/.github/actions/prepare-case" + with: + case-id: obs-studio + - name: Read pinned source + id: source + run: sed '/^$/d' benchmark-source.env >> "$GITHUB_OUTPUT" + - name: Checkout pinned OBS source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + with: + repository: "${{ steps.source.outputs.OBS_SOURCE_REPOSITORY }}" + ref: "${{ steps.source.outputs.OBS_HEAD_SHA }}" + path: upstream + fetch-depth: 0 + submodules: recursive + persist-credentials: false + - name: Prepare rolling source and matching Xcode cohort + run: | + ./scripts/prepare-source.sh rolling + ruby scripts/scope-case-cache.rb "obs-studio-$CACHE_COHORT" + sudo xcode-select --switch "${{ steps.source.outputs.OBS_XCODE_PATH }}" + - name: Start cache setup timing + id: restore_start + run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" + - name: Restore and publish BoringCache xcode + id: cache + uses: "./.github/actions/boringcache" + with: + cli-version: "${{ inputs.cli_version }}" + trust-policy: publish + mode: xcode + working-directory: "./upstream" + - name: Capture cache setup timing + id: restore + run: echo "seconds=$(( $(date +%s) - ${{ steps.restore_start.outputs.started_at + }} ))" >> "$GITHUB_OUTPUT" + - name: Build rolling commit + id: build + run: | + set -euo pipefail + ruby ./scripts/measure-build.rb --output benchmark-results/xcode-boringcache-rolling-timing.json -- ruby ./scripts/run-benchmark-plan.rb xcode --working-directory upstream \ + 2>&1 | tee benchmark-results/xcode-boringcache-rolling.log + ruby -rjson -e 'puts "seconds=#{JSON.parse(File.read(ARGV.fetch(0))).fetch("build_seconds")}"' benchmark-results/xcode-boringcache-rolling-timing.json >> "$GITHUB_OUTPUT" + - name: Require native Xcode build evidence + run: "./scripts/verify_xcode_build_log.rb benchmark-results/xcode-boringcache-rolling.log" + - name: Write rolling result + run: | + ./scripts/write_phase_result.rb \ + --surface xcode \ + --strategy boringcache \ + --phase rolling --continuous --cache-hit "$RESTORED_CACHE_HIT" \ + --source-sha "${{ steps.source.outputs.OBS_HEAD_SHA }}" \ + --restore-seconds "${{ steps.restore.outputs.seconds }}" \ + --build-seconds "${{ steps.build.outputs.seconds }}" \ + --action-evidence "${{ steps.cache.outputs.evidence-path }}" + env: + RESTORED_CACHE_HIT: "${{ steps.cache.outputs.cache-hit }}" + - name: Upload rolling result + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: phase-obs-studio-boringcache-xcode-rolling-commit + path: benchmark-results + if-no-files-found: error + retention-days: 30 + if: always() + - name: Retain BoringCache product evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: boringcache-product-xcode-rolling + path: "${{ steps.cache.outputs.evidence-path }}" + if-no-files-found: error + retention-days: 30 + if: always() + permissions: + actions: read + contents: read + id-token: write + env: + BENCHMARK_RUNNER_CLASS: macos-26 + report: + name: obs-studio rolling report + needs: + - actions-cache-ccache + - actions-cache-xcode + - boringcache-ccache + - boringcache-xcode + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-obs-studio-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: obs-studio rolling benchmark + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain actions-cache ccache result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'ccache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-actions-cache-ccache-rolling + path: benchmark-results/obs-studio-actions-cache-ccache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain actions-cache xcode result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'xcode' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-actions-cache-xcode-rolling + path: benchmark-results/obs-studio-actions-cache-xcode-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain boringcache ccache result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'ccache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-boringcache-ccache-rolling + path: benchmark-results/obs-studio-boringcache-ccache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain boringcache xcode result + if: inputs.cache_tool == 'all' || inputs.cache_tool == 'xcode' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-obs-studio-boringcache-xcode-rolling + path: benchmark-results/obs-studio-boringcache-xcode-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-obs-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/reapi-fresh-benchmark.yml b/.github/workflows/reapi-fresh-benchmark.yml index 733cbdba2..b1b322b74 100644 --- a/.github/workflows/reapi-fresh-benchmark.yml +++ b/.github/workflows/reapi-fresh-benchmark.yml @@ -1,33 +1,13 @@ +--- name: REAPI comparison fresh -run-name: '${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample }}' -on: - workflow_dispatch: - inputs: - case_id: - type: choice - options: [gogs-moon, opencut-moon, stackstorm-pants, executorch-buck2, msgpack-sbt] - required: true - provider: - description: Selected provider for a native cache correctness screen. - type: choice - options: [all, boringcache, bazel-remote] - default: all - cli_version: - description: Exact CLI release or canary; required for BoringCache execution. - type: string - default: '' - series_id: - type: string - default: '' - sample: - type: string - default: '1' +run-name: "${{ inputs.case_id }} fresh ${{ inputs.series_id }} sample ${{ inputs.sample + }}" permissions: contents: read actions: read env: - BENCHMARK_SERIES_ID: ${{ inputs.series_id }} - BENCHMARK_SAMPLE: ${{ inputs.sample }} + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" BENCHMARK_RUNNER_CLASS: ubuntu-24.04 BORINGCACHE_NO_GIT: 'true' jobs: @@ -35,20 +15,21 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: - matrix: ${{ steps.providers.outputs.matrix }} + matrix: "${{ steps.providers.outputs.matrix }}" steps: - - id: providers - env: - PROVIDER: ${{ inputs.provider }} - run: | - case "$PROVIDER" in - all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; - boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; - bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + run: | + case "$PROVIDER" in + all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac cold: - name: '${{ matrix.provider }} ${{ inputs.case_id }} cold' + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} cold" needs: providers runs-on: ubuntu-24.04 timeout-minutes: 180 @@ -59,24 +40,27 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/reapi-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: cold - cli-version: ${{ inputs.cli_version }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: cold + cli-version: "${{ inputs.cli_version }}" warm: - name: '${{ matrix.provider }} ${{ inputs.case_id }} warm' - needs: [providers, cold] + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} warm" + needs: + - providers + - cold runs-on: ubuntu-24.04 timeout-minutes: 180 permissions: @@ -86,18 +70,94 @@ jobs: strategy: fail-fast: false matrix: - provider: ${{ fromJSON(needs.providers.outputs.matrix) }} + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: warm + cli-version: "${{ inputs.cli_version }}" + report: + name: "${{ inputs.case_id }} fresh report" + needs: + - cold + - warm + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - path: .harness - persist-credentials: false - - uses: ./.harness/.github/actions/prepare-case - with: - case-id: ${{ inputs.case_id }} - - uses: ./.github/actions/reapi-benchmark - with: - case-id: ${{ inputs.case_id }} - provider: ${{ matrix.provider }} - phase: warm - cli-version: ${{ inputs.cli_version }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} fresh benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-fresh + path: benchmark-results/${{ inputs.case_id }}-boringcache-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain bazel-remote result + if: inputs.provider == 'all' || inputs.provider == 'bazel-remote' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-bazel-remote-fresh + path: benchmark-results/${{ inputs.case_id }}-bazel-remote-fresh.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-reapi-fresh + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - gogs-moon + - opencut-moon + - stackstorm-pants + - executorch-buck2 + - msgpack-sbt + required: true + provider: + description: Selected provider for a native cache correctness screen. + type: choice + options: + - all + - boringcache + - bazel-remote + default: all + cli_version: + description: Exact CLI release or canary; required for BoringCache execution. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' diff --git a/.github/workflows/reapi-rolling-benchmark.yml b/.github/workflows/reapi-rolling-benchmark.yml new file mode 100644 index 000000000..178ca1d4a --- /dev/null +++ b/.github/workflows/reapi-rolling-benchmark.yml @@ -0,0 +1,142 @@ +--- +name: REAPI comparison rolling +run-name: "${{ inputs.case_id }} rolling ${{ inputs.series_id }} sample ${{ inputs.sample + }}" +permissions: + contents: read + actions: read +env: + BENCHMARK_SERIES_ID: "${{ inputs.series_id }}" + BENCHMARK_SAMPLE: "${{ inputs.sample }}" + BENCHMARK_RUNNER_CLASS: ubuntu-24.04 + BORINGCACHE_NO_GIT: 'true' + BENCHMARK_ROLLING_SCOPE: "${{ inputs.cache_scope }}" +jobs: + providers: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: "${{ steps.providers.outputs.matrix }}" + steps: + - id: providers + env: + PROVIDER: "${{ inputs.provider }}" + run: | + case "$PROVIDER" in + all) echo 'matrix=["boringcache","bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + boringcache) echo 'matrix=["boringcache"]' >> "$GITHUB_OUTPUT" ;; + bazel-remote) echo 'matrix=["bazel-remote"]' >> "$GITHUB_OUTPUT" ;; + *) exit 1 ;; + esac + commit: + name: "${{ matrix.provider == 'boringcache' && 'BoringCache' || matrix.provider + == 'cachix' && 'Cachix' || matrix.provider }} ${{ inputs.case_id }} commit" + needs: providers + runs-on: ubuntu-24.04 + timeout-minutes: 180 + permissions: + contents: read + actions: read + id-token: write + strategy: + fail-fast: false + matrix: + provider: "${{ fromJSON(needs.providers.outputs.matrix) }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: ".harness" + persist-credentials: false + - uses: "./.harness/.github/actions/prepare-case" + with: + case-id: "${{ inputs.case_id }}" + - uses: "./.github/actions/reapi-benchmark" + with: + case-id: "${{ inputs.case_id }}" + provider: "${{ matrix.provider }}" + phase: commit + cli-version: "${{ inputs.cli_version }}" + lane: rolling + report: + name: "${{ inputs.case_id }} rolling report" + needs: + - commit + if: always() && !cancelled() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - name: Download verified phase evidence + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + pattern: phase-${{ inputs.case_id }}-* + path: phase-evidence + - name: Generate the benchmark report + env: + REPORT_TITLE: "${{ inputs.case_id }} rolling benchmark" + run: ruby scripts/canonical/benchmark-report.rb summarize --title "$REPORT_TITLE" + --input-dir phase-evidence --output-dir benchmark-results + - name: Retain boringcache result + if: inputs.provider == 'all' || inputs.provider == 'boringcache' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-boringcache-rolling + path: benchmark-results/${{ inputs.case_id }}-boringcache-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain bazel-remote result + if: inputs.provider == 'all' || inputs.provider == 'bazel-remote' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: benchmark-${{ inputs.case_id }}-bazel-remote-rolling + path: benchmark-results/${{ inputs.case_id }}-bazel-remote-rolling.json + if-no-files-found: error + retention-days: 90 + - name: Retain comparison summary + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: comparison-reapi-rolling + path: benchmark-results/comparison.md + if-no-files-found: error + retention-days: 90 +concurrency: + group: benchmark-reapi-rolling-${{ inputs.case_id }}-${{ inputs.cache_scope || inputs.series_id + || github.ref_name }} + cancel-in-progress: false + queue: max +'on': + workflow_dispatch: + inputs: + case_id: + type: choice + options: + - gogs-moon + - opencut-moon + - stackstorm-pants + - executorch-buck2 + - msgpack-sbt + required: true + provider: + description: Selected provider for a native cache correctness screen. + type: choice + options: + - all + - boringcache + - bazel-remote + default: all + cli_version: + description: Exact CLI release or canary; required for BoringCache execution. + type: string + default: '' + series_id: + type: string + default: '' + sample: + type: string + default: '1' + cache_scope: + type: string + default: '' diff --git a/.github/workflows/source-case.yml b/.github/workflows/source-case.yml new file mode 100644 index 000000000..aa6181475 --- /dev/null +++ b/.github/workflows/source-case.yml @@ -0,0 +1,121 @@ +name: Check and queue one rolling case +on: + workflow_call: + inputs: + case_id: + type: string + required: true +permissions: + contents: read + actions: read +jobs: + previous: + name: '${{ inputs.case_id }} previous rolling run' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + actions: read + outputs: + proceed: ${{ steps.previous.outputs.proceed }} + ref: ${{ steps.previous.outputs.ref }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - id: previous + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + BENCHMARK_CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + if [[ "$BENCHMARK_CADENCE_ACTIVE" == true && "$GITHUB_REF_NAME" == main ]]; then + ruby scripts/source-promotion.rb --case "$CASE_ID" --reconcile \ + --output "$RUNNER_TEMP/rolling-previous.json" + else + echo 'proceed=true' >> "$GITHUB_OUTPUT" + echo "ref=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + fi + - name: Retain previous rolling state + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: rolling-previous-${{ inputs.case_id }} + path: ${{ runner.temp }}/rolling-previous.json + if-no-files-found: ignore + retention-days: 90 + inspect: + name: '${{ inputs.case_id }} source check' + needs: previous + if: needs.previous.outputs.proceed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + outputs: + changed: ${{ steps.proposal.outputs.changed }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ needs.previous.outputs.ref }} + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - name: Inspect upstream against the reviewed recipe + id: proposal + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + run: | + ruby scripts/sync-sources.rb --case "$CASE_ID" --output "$RUNNER_TEMP/source-proposals.json" + echo "changed=$(ruby -rjson -e 'puts JSON.parse(File.read(ARGV[0])).fetch("records").first.fetch("state") == "proposal"' "$RUNNER_TEMP/source-proposals.json")" >> "$GITHUB_OUTPUT" + - name: Retain the source observation + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: source-proposals-${{ inputs.case_id }} + path: ${{ runner.temp }}/source-proposals.json + if-no-files-found: error + retention-days: 90 + publish: + name: '${{ inputs.case_id }} queue rolling runs' + needs: [previous, inspect] + if: needs.inspect.outputs.changed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + actions: write + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + ref: ${{ needs.previous.outputs.ref }} + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: source-proposals-${{ inputs.case_id }} + path: ${{ runner.temp }}/proposal + - name: Publish source and request the declared rolling variants + env: + GH_TOKEN: ${{ github.token }} + CASE_ID: ${{ inputs.case_id }} + BENCHMARK_CADENCE_ACTIVE: ${{ vars.BENCHMARK_CADENCE_ACTIVE }} + run: | + args=(--channel canary) + if [[ "$BENCHMARK_CADENCE_ACTIVE" == true && "$GITHUB_REF_NAME" == main ]]; then + args=(--channel stable --publish) + fi + ruby scripts/source-promotion.rb --case "$CASE_ID" "${args[@]}" \ + --proposal "$RUNNER_TEMP/proposal/source-proposals.json" \ + --output "$RUNNER_TEMP/rolling-dispatch.json" + - name: Retain rolling dispatch receipt + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: rolling-dispatch-${{ inputs.case_id }} + path: ${{ runner.temp }}/rolling-dispatch.json + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/source-sync.yml b/.github/workflows/source-sync.yml index 42a8bbc0e..c13b89116 100644 --- a/.github/workflows/source-sync.yml +++ b/.github/workflows/source-sync.yml @@ -1,41 +1,35 @@ +--- name: Check upstream source updates -on: - schedule: - - cron: '11 * * * *' - workflow_dispatch: - inputs: - case_id: - description: Optional case ID; empty checks the scheduled suite. - type: string - default: '' permissions: contents: read actions: read jobs: select: - if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE == 'true' + if: github.event_name == 'workflow_dispatch' || vars.BENCHMARK_CADENCE_ACTIVE + == 'true' runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: - matrix: ${{ steps.cases.outputs.matrix }} + matrix: "${{ steps.cases.outputs.matrix }}" steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - run: bundle install --jobs 4 --retry 2 - - id: cases - env: - CASE_ID: ${{ inputs.case_id }} - run: | - args=() - [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") - echo "matrix=$(ruby scripts/sync-sources.rb --matrix "${args[@]}")" >> "$GITHUB_OUTPUT" - propose: - name: '${{ matrix.case_id }} source check' + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b + - run: bundle install --jobs 4 --retry 2 + - id: cases + env: + CASE_ID: "${{ inputs.case_id }}" + run: | + args=() + [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") + echo "matrix=$(ruby scripts/sync-sources.rb --matrix "${args[@]}")" >> "$GITHUB_OUTPUT" + sources: + name: "${{ matrix.case_id }}" needs: select - runs-on: ubuntu-24.04 - timeout-minutes: 60 + permissions: + contents: write + actions: write concurrency: group: benchmark-source-${{ matrix.case_id }} cancel-in-progress: false @@ -43,32 +37,16 @@ jobs: strategy: fail-fast: false max-parallel: 6 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 - with: - persist-credentials: false - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - - run: bundle install --jobs 4 --retry 2 - - name: Prepare reviewed source proposals - env: - GH_TOKEN: ${{ github.token }} - CASE_ID: ${{ matrix.case_id }} - run: | - args=() - [[ -z "$CASE_ID" ]] || args+=(--case "$CASE_ID") - ruby scripts/sync-sources.rb "${args[@]}" --output "$RUNNER_TEMP/source-proposals.json" - - name: Retain proposed definitions and qualification requirements - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f - with: - name: source-proposals-${{ matrix.case_id }} - path: | - ${{ runner.temp }}/source-proposals.json - cases/${{ matrix.case_id }}/case.json - cases/${{ matrix.case_id }}/payload/benchmark-source.env - cases/${{ matrix.case_id }}/payload/recipe-contract.json - cases/${{ matrix.case_id }}/payload/.boringcache.toml - include-hidden-files: true - if-no-files-found: error - retention-days: 90 + matrix: "${{ fromJSON(needs.select.outputs.matrix) }}" + uses: "./.github/workflows/source-case.yml" + with: + case_id: "${{ matrix.case_id }}" +'on': + schedule: + - cron: 11 * * * * + workflow_dispatch: + inputs: + case_id: + description: Optional case ID; empty checks the scheduled suite. + type: string + default: '' diff --git a/.github/workflows/zed-zed-cargo-rolling-auto.yml b/.github/workflows/zed-zed-cargo-rolling-auto.yml index 6c0b2f5e8..216471156 100644 --- a/.github/workflows/zed-zed-cargo-rolling-auto.yml +++ b/.github/workflows/zed-zed-cargo-rolling-auto.yml @@ -166,7 +166,7 @@ jobs: permissions: actions: read checks: read - contents: write + contents: read id-token: write publish-source: name: Publish the verified Zed source diff --git a/cases/executorch-buck2/case.json b/cases/executorch-buck2/case.json index dd69ab280..c0ebad736 100644 --- a/cases/executorch-buck2/case.json +++ b/cases/executorch-buck2/case.json @@ -30,6 +30,13 @@ "inputs": { "case_id": "executorch-buck2" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "executorch-buck2" + } } ], "blockers": [] @@ -51,9 +58,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/gogs-moon/case.json b/cases/gogs-moon/case.json index 274eec4e1..de759e8ff 100644 --- a/cases/gogs-moon/case.json +++ b/cases/gogs-moon/case.json @@ -29,6 +29,13 @@ "inputs": { "case_id": "gogs-moon" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "gogs-moon" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/gogs-moon/payload/recipe-contract.json b/cases/gogs-moon/payload/recipe-contract.json index dfb80366b..148157c28 100644 --- a/cases/gogs-moon/payload/recipe-contract.json +++ b/cases/gogs-moon/payload/recipe-contract.json @@ -6,5 +6,15 @@ "pnpm-lock.yaml": "eb8bbc6812fbdfb3c847d99d4af09925e9b4c6b285e99472c4193d483d90f523", "web/package.json": "e8e4ea83c1bdb84effa1ec14c45d63024e0f13b197a36847b8680dfd0b57aeb4" }, - "commands": {} + "commands": { + }, + "upstream_files_by_revision": { + "103893ed5f78d2f138f5286a2dd3b4e94d18fb0b": { + ".moon/workspace.yml": "74f75a4b2cc84d452c7911159396fc28ec858e67c6033a54ec30baa12696d73f", + "moon.yml": "e2c40b021b55a6a0c5de5c1de90bf659471cacfdedf684dd318ebfac74ff2616", + "web/moon.yml": "e800f64593003a6cdb5cd0cce17da3617b811bb712eb8263731e02de3a164cd2", + "pnpm-lock.yaml": "801e39410e1ee70f9d1b5872d17fc6e7b103fe1b6335d185b5d6cb8d61e3df2b", + "web/package.json": "01e0ab93e1abf863194730dd6ea4bf84c3c7ee1a0b4cd89ec098956528f0a2ac" + } + } } diff --git a/cases/helix-nix/case.json b/cases/helix-nix/case.json index f23e2d32b..aa1dc8231 100644 --- a/cases/helix-nix/case.json +++ b/cases/helix-nix/case.json @@ -30,6 +30,13 @@ "case_id": "helix-nix", "provider": "all" } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "helix-nix" + } } ], "blockers": [], @@ -55,9 +62,9 @@ "cachix" ], "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/immich/payload/recipe-contract.json b/cases/immich/payload/recipe-contract.json index c8f263b77..857b05ebd 100644 --- a/cases/immich/payload/recipe-contract.json +++ b/cases/immich/payload/recipe-contract.json @@ -1,8 +1,8 @@ { "schema_version": 1, "upstream_files": { - ".github/workflows/docker.yml": "79f9b93dc0bdfc5c9e55550331522837329ce061bebfa5737d97a27ea3b4b795", - "server/Dockerfile": "b64d6daffd88147d57fae1ff115fe92f520b6df9a6f33a8da66648f80a0821da" + ".github/workflows/docker.yml": "2545e683c5bb11ab0e2ab9a706289c0603be5ded38a560dc6faf57528dca5cd9", + "server/Dockerfile": "fd0231bdac647f70c31caf0ff22a23bbfd8fb74666d3d52cb25d8ea8fd819757" }, "commands": { "docker": [ @@ -27,5 +27,11 @@ "immich-server-benchmark:local", "upstream" ] + }, + "upstream_files_by_revision": { + "007c18aec6cc9552fb7d8b7c791c41d7424876b0": { + ".github/workflows/docker.yml": "79f9b93dc0bdfc5c9e55550331522837329ce061bebfa5737d97a27ea3b4b795", + "server/Dockerfile": "b64d6daffd88147d57fae1ff115fe92f520b6df9a6f33a8da66648f80a0821da" + } } } diff --git a/cases/msgpack-sbt/case.json b/cases/msgpack-sbt/case.json index d6cb644c4..07e6d7fae 100644 --- a/cases/msgpack-sbt/case.json +++ b/cases/msgpack-sbt/case.json @@ -29,6 +29,13 @@ "inputs": { "case_id": "msgpack-sbt" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "msgpack-sbt" + } } ], "sync": "upstream-head", @@ -50,10 +57,10 @@ "boringcache", "bazel-remote" ], - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "primary_metric": "correctness", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/msgpack-sbt/payload/recipe-contract.json b/cases/msgpack-sbt/payload/recipe-contract.json index 3b9f981bd..e75a08504 100644 --- a/cases/msgpack-sbt/payload/recipe-contract.json +++ b/cases/msgpack-sbt/payload/recipe-contract.json @@ -1,9 +1,18 @@ { "upstream_files": { "build.sbt": "c4e03e280c4da5cdc930759bf7fdd368c5d30999290aded4cd636042ebc0e89c", - "project/build.properties": "331ebccd1dbd48154f6c230393795dd72d370e6fdb9952ff2dd22b9eae4902b2", + "project/build.properties": "94a37bc3ad2e038979be0b7172c35904a468ab8e6e9c61e0993dd95af77b23ce", "project/plugins.sbt": "eab49425b64f57fcdfaef54f4ec1722e78165a86e51ece33751566adcb7db6ac", "sbt": "b7cd1f08b549fc67528c9d6a6a026a3342e82ba46dc5dc8a98729ceb201d0581" }, - "commands": {} + "commands": { + }, + "upstream_files_by_revision": { + "5c28fbad6cd360d8ea822df109830ae2517a7ae4": { + "build.sbt": "c4e03e280c4da5cdc930759bf7fdd368c5d30999290aded4cd636042ebc0e89c", + "project/build.properties": "331ebccd1dbd48154f6c230393795dd72d370e6fdb9952ff2dd22b9eae4902b2", + "project/plugins.sbt": "eab49425b64f57fcdfaef54f4ec1722e78165a86e51ece33751566adcb7db6ac", + "sbt": "b7cd1f08b549fc67528c9d6a6a026a3342e82ba46dc5dc8a98729ceb201d0581" + } + } } diff --git a/cases/obs-studio/case.json b/cases/obs-studio/case.json index b4daea835..8f89c2025 100644 --- a/cases/obs-studio/case.json +++ b/cases/obs-studio/case.json @@ -30,7 +30,8 @@ "path": ".github/workflows/obs-studio-obs-actions-cache.yml", "variant_input": "cache_tool", "lane": "fresh", - "inputs": {}, + "inputs": { + }, "phases": [ "cold", "commit" @@ -44,7 +45,8 @@ "path": ".github/workflows/obs-studio-obs-boringcache.yml", "variant_input": "cache_tool", "lane": "fresh", - "inputs": {}, + "inputs": { + }, "phases": [ "cold", "commit" @@ -57,15 +59,33 @@ { "path": ".github/workflows/obs-studio-obs-xcode-continuation.yml", "lane": "diagnostic", - "inputs": {} + "inputs": { + } + }, + { + "path": ".github/workflows/obs-rolling-benchmark.yml", + "variant_input": "cache_tool", + "lane": "rolling", + "inputs": { + }, + "variants": [ + "ccache", + "xcode" + ] } ], - "sync": "adjacent-pair", + "sync": "build-relevant-pair", + "cache_tags": { + "obs-studio-ccache": "{scope}-ccache", + "obs-studio-xcode": "{scope}-xcode" + }, "sync_minute": 18, "source_prefix": "OBS" }, "reporting": { - "entries": [], + "entries": [ + + ], "publication": "reviewed" }, "verification": [ diff --git a/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb b/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb index b537f232f..7169b4661 100755 --- a/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb +++ b/cases/obs-studio/payload/scripts/verify_ccache_evidence.rb @@ -7,14 +7,19 @@ %w[path phase strategy].each { |name| parser.on("--#{name} VALUE") { |value| args[name] = value } } end.parse! payload = JSON.parse(File.read(args.fetch("path"))) -raise "Unknown phase or strategy" unless %w[base rolling].include?(args["phase"]) && %w[actions-cache boringcache].include?(args["strategy"]) +raise "Unknown phase or strategy" unless %w[base rolling commit].include?(args["phase"]) && %w[actions-cache boringcache].include?(args["strategy"]) errors, timeouts = payload.values_at("remote_storage_error", "remote_storage_timeout").map(&:to_i) raise "ccache reported remote errors=#{errors}, timeouts=#{timeouts}" unless errors.zero? && timeouts.zero? hits = payload.fetch("direct_cache_hit", 0) + payload.fetch("preprocessed_cache_hit", 0) base = args["phase"] == "base" raise "Cold ccache build did not report cache misses" if base && payload.fetch("cache_miss", 0) <= 0 -raise "Rolling ccache build did not report cache hits" if !base && hits <= 0 +raise "Rolling ccache build did not report cache hits" if args["phase"] == "rolling" && hits <= 0 +raise "ccache build did not report compilation" if args["phase"] == "commit" && hits + payload.fetch("cache_miss", 0) <= 0 if args["strategy"] == "boringcache" - field = base ? "remote_storage_write" : "remote_storage_hit" - raise "BoringCache did not report #{field}" if payload.fetch(field, 0) <= 0 + if args["phase"] == "commit" + raise "BoringCache did not report a remote read or write" if payload.fetch("remote_storage_hit", 0) + payload.fetch("remote_storage_write", 0) <= 0 + else + field = base ? "remote_storage_write" : "remote_storage_hit" + raise "BoringCache did not report #{field}" if payload.fetch(field, 0) <= 0 + end end diff --git a/cases/obs-studio/payload/scripts/write_phase_result.rb b/cases/obs-studio/payload/scripts/write_phase_result.rb index 9554e7aae..5ef340509 100755 --- a/cases/obs-studio/payload/scripts/write_phase_result.rb +++ b/cases/obs-studio/payload/scripts/write_phase_result.rb @@ -6,12 +6,16 @@ args = {"output_dir" => "benchmark-results", "cache_key" => ""} OptionParser.new do |parser| - %w[surface strategy phase source_sha restore_seconds build_seconds cache_key action_evidence native_evidence output_dir].each do |name| + %w[surface strategy phase source_sha restore_seconds build_seconds cache_key cache_hit action_evidence native_evidence output_dir].each do |name| parser.on("--#{name.tr('_','-')} VALUE") { |value| args[name] = value } end + parser.on("--continuous") { args["continuous"] = true } end.parse! phase = args.fetch("phase") raise "Phase must be base or rolling" unless %w[base rolling].include?(phase) +raise "Continuous observations require the rolling phase" if args["continuous"] && phase != "rolling" +cache_hit = args["continuous"] ? args["cache_hit"] : (phase == "base" ? "false" : "true") +raise "Cache hit must be true or false" if cache_hit && !%w[true false].include?(cache_hit) build_seconds = Float(args.fetch("build_seconds")) raise "Build time must be finite and nonnegative" unless build_seconds.finite? && build_seconds >= 0 native = args["native_evidence"] && JSON.parse(File.read(args["native_evidence"])) @@ -31,10 +35,14 @@ "remote_storage_hits" => native.fetch("remote_storage_hit", 0), "remote_storage_writes" => native.fetch("remote_storage_write", 0), "remote_storage_errors" => native.fetch("remote_storage_error", 0), "remote_storage_timeouts" => native.fetch("remote_storage_timeout", 0)}, "github" => BenchmarkReport.github_identity} +if args["continuous"] + result["classification"]["reporting_reason"] = "Source build using the persistent compiler-cache cohort" + result["classification"]["cache_import_status"] = {"true" => "hit", "false" => "miss"}.fetch(cache_hit, "unmeasured") +end BenchmarkReport.write_json(File.join(args["output_dir"], "#{args['surface']}-#{args['strategy']}-#{phase}.json"), result) -BenchmarkReport.phase({"benchmark" => "obs-studio-#{args['surface']}", "strategy" => args["strategy"], "variant" => args.fetch("surface"), +BenchmarkReport.phase({"benchmark" => args["continuous"] ? "obs-studio" : "obs-studio-#{args['surface']}", "strategy" => args["strategy"], "variant" => args.fetch("surface"), "lane" => phase == "base" ? "fresh" : "rolling", "phase" => phase == "base" ? "cold" : "commit", "mode" => args["surface"], "source_repository" => "obsproject/obs-studio", "source_sha" => args["source_sha"], "build_seconds" => build_seconds, - "restore_or_setup_seconds" => Integer(args["restore_seconds"]), "cache_hit" => phase == "base" ? "false" : "true", + "restore_or_setup_seconds" => Integer(args["restore_seconds"]), "cache_hit" => cache_hit, "evidence" => args["action_evidence"], "storage_key" => args["cache_key"], "output_dir" => args["output_dir"], "verification_passed" => true, "comparison_seconds" => Integer(args["restore_seconds"]) + build_seconds}) diff --git a/cases/opencut-moon/case.json b/cases/opencut-moon/case.json index 3f8cfa70e..1471e64a6 100644 --- a/cases/opencut-moon/case.json +++ b/cases/opencut-moon/case.json @@ -29,6 +29,13 @@ "inputs": { "case_id": "opencut-moon" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "opencut-moon" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/qdrant/payload/recipe-contract.json b/cases/qdrant/payload/recipe-contract.json index f5ea1ba36..707ff2586 100644 --- a/cases/qdrant/payload/recipe-contract.json +++ b/cases/qdrant/payload/recipe-contract.json @@ -1,7 +1,7 @@ { "schema_version": 1, "upstream_files": { - ".github/workflows/integration-tests.yml": "b8ce312d966388b4ea8b748eedc9ad61d15934d684ae06e4cf29b546a5084418", + ".github/workflows/integration-tests.yml": "ca7026d1a1e15e45bd40cf92dbc4a44828b374e3fae71cbf64fc70d830c871cf", "Dockerfile": "e5796337f4c1fa080ddad9dd4260e5b8eec9dc6920ded0b4a3332e73bfb1db7f" }, "commands": { @@ -20,5 +20,11 @@ "qdrant/qdrant:e2e-tests", "upstream" ] + }, + "upstream_files_by_revision": { + "9ae05efdc1ef96352889b2a6b6b574c35232b974": { + ".github/workflows/integration-tests.yml": "b8ce312d966388b4ea8b748eedc9ad61d15934d684ae06e4cf29b546a5084418", + "Dockerfile": "e5796337f4c1fa080ddad9dd4260e5b8eec9dc6920ded0b4a3332e73bfb1db7f" + } } } diff --git a/cases/stackstorm-pants/case.json b/cases/stackstorm-pants/case.json index 78e123d87..1cc500ef1 100644 --- a/cases/stackstorm-pants/case.json +++ b/cases/stackstorm-pants/case.json @@ -29,6 +29,13 @@ "inputs": { "case_id": "stackstorm-pants" } + }, + { + "path": ".github/workflows/reapi-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "stackstorm-pants" + } } ], "blockers": [] @@ -50,9 +57,9 @@ "bazel-remote" ], "primary_metric": "correctness", - "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance.", + "timed_scope": "Native command and registry startup after dependency preparation; registry shutdown is recorded separately. The bazel-remote store is transferred as a GitHub artifact outside timing. This screen does not compare end-to-end provider performance. Rolling runs use the same measurement boundary. Comparator cache transfer is outside build timing; rolling reuse requires retained seed and native hit evidence.", "storage": "unmeasured", - "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker.", + "cache_scope": "BoringCache uses a fresh case, series, sample and run tag. bazel-remote uses a fresh local store transferred to a new warm worker. Native local result state starts empty on each worker. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/zed-nix/case.json b/cases/zed-nix/case.json index 82135887d..87058a727 100644 --- a/cases/zed-nix/case.json +++ b/cases/zed-nix/case.json @@ -30,6 +30,13 @@ "case_id": "zed-nix", "provider": "all" } + }, + { + "path": ".github/workflows/nix-rolling-benchmark.yml", + "lane": "rolling", + "inputs": { + "case_id": "zed-nix" + } } ], "blockers": [], @@ -55,9 +62,9 @@ "cachix" ], "primary_metric": "build_and_reuse_seconds", - "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", "storage": "provider-reported", - "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", "sample_count": 2 } } diff --git a/cases/zed/case.json b/cases/zed/case.json index 57eb38fd4..b5c4d0409 100644 --- a/cases/zed/case.json +++ b/cases/zed/case.json @@ -76,6 +76,7 @@ } ], "sync": "verified-pair", + "rolling_workflow": ".github/workflows/zed-zed-cargo-rolling-auto.yml", "sync_minute": 28, "source_prefix": "ZED", "cache_tags": { diff --git a/cases/zed/payload/recipe-contract.json b/cases/zed/payload/recipe-contract.json index c723c1b25..8add934c2 100644 --- a/cases/zed/payload/recipe-contract.json +++ b/cases/zed/payload/recipe-contract.json @@ -2,7 +2,7 @@ "schema_version": 1, "upstream_files": { "rust-toolchain.toml": "3d9002e97d5c3de93d49a1a36649153da4f40c29b061d0f2e09ea84ee143ba3e", - "script/bundle-linux": "6c8ceb443cb364259aeadebbfb47dba829cef81fccef304bd19d20724f1f0be7" + "script/bundle-linux": "8d269b2d6f304440f3a0951d5dfd79837c08fb2cf2bd5e8fbf59bb589428e43d" }, "commands": { "cargo": [ @@ -11,5 +11,11 @@ "--release", "--locked" ] + }, + "upstream_files_by_revision": { + "54edefa0832e4728bcf077e4a48b25912eea5433": { + "rust-toolchain.toml": "3d9002e97d5c3de93d49a1a36649153da4f40c29b061d0f2e09ea84ee143ba3e", + "script/bundle-linux": "6c8ceb443cb364259aeadebbfb47dba829cef81fccef304bd19d20724f1f0be7" + } } } diff --git a/docs/cadence.md b/docs/cadence.md index eea4d5131..7f891dcbb 100644 --- a/docs/cadence.md +++ b/docs/cadence.md @@ -1,84 +1,105 @@ # Benchmark schedules -The schedule migration is incomplete. Historical repositories still own the -active weekly, canary and upstream-sync schedules. Do not enable duplicate -central schedules before removing the corresponding historical triggers. -Central scheduled jobs require the repository variable -`BENCHMARK_CADENCE_ACTIVE=true`. Manual checks remain available while it is -unset. Canary monitoring continues to inspect the historical repositories until -that same switch changes ownership. +Central activation remains off. Historical repositories still own the active +weekly, canary and source schedules. Central scheduled jobs require +`BENCHMARK_CADENCE_ACTIVE=true`; merging this branch does not activate them. +Manual qualification remains available. Historical triggers must be retired +before ownership changes, so the migration does not duplicate runs. -`suites/scheduled.json` selects the maintained suite and the new Moon, Pants, -Buck2, sbt and Nix workloads. It excludes Docker corpus and prospect drafts. -The same selections feed fresh dispatch and source checks. Case definitions own -workflow paths and variant inputs; the suite does not copy them. - -The proposed central cadence is: +`suites/scheduled.json` selects the maintained suite plus Moon, Pants, Buck2, +sbt and Nix: 24 source cases and 31 fresh targets. It excludes Docker corpus and +prospect drafts. Case definitions own workflow paths and variant inputs. | Run | UTC schedule | CLI | Source | | --- | --- | --- | --- | -| Weekly fresh | Monday 04:00 | Latest published stable release, with all required assets | Declared pins | -| Nightly fresh | Daily 01:17 | Latest published canary, with all required assets | Declared pins | -| Source check | Hourly, at :11 | No build | Check upstream against declared pins | - -Fresh dispatch resolves one exact release tag for every selected workflow. It -validates all targets before dispatch and retains each requested run ID. A -matrix requests each target independently; a failed request does not prevent -other targets from being requested. Each target retains its own receipt, and -the combined receipt keeps failed, uncertain and missing requests visible. A -dispatch succeeding does not establish that its builds passed. A dry run starts -no builds. Inspect the retained receipt before retrying an interrupted dispatch. -When the selection includes REAPI cases, preflight downloads the exact CLI, -verifies its release checksum, and requires `cache-registry --reapi-port` in its -command interface before dispatching any target. This check also runs in dry runs. - -The October 5 capability check rejected stable `v1.33.0` before dispatch: it -does not expose `--reapi-port`. Canary `vcli-canary-7a5b27146ebe` passed the same -31-target check. Stable activation remains pending a compatible release; the -weekly caller must not silently substitute a canary or omit the new families. - -Source checks currently retain proposals only. The seven new snapshot cases now -check their declared upstream branches, require fast-forward ancestry, and verify -the reviewed recipe against an exact candidate commit. Nix proposals update the -matching source URI in the plan and command contract; they do not change recipe -digests or build flags. A recipe or patch failure leaves the declared pin unchanged. -An unchanged source is recorded as unchanged; a recipe mismatch or fixed source -is recorded as blocked. A later -failure does not remove earlier case checks. The workflow does not commit source -changes, promote candidates or start rolling builds. -Each case has an independent job and artifact. A failed or slow source check -does not prevent the other cases from being checked. Checks for the same case -are serialized. - -Immich, Mastodon, PostHog and n8n now register the shared native rolling workflow. -Their Docker variants load and inspect output in both providers. The same output -contract now applies to Hugo, Chroma, Duckgres and Linkerd2 rolling definitions, -which previously selected image publication. These definitions need live rolling -qualification; passing schema and workflow checks does not establish seed reuse. - -The repository's Actions cache limit was increased from 10 GB to 200 GB on -October 5. Retention is still seven days. This provides more capacity, but does -not prove that an individual rolling seed was available or restored. - -## Work remaining before schedule migration - -- Connect reviewed source advancement to rolling dispatch and retain the exact - parent source, cache seed and requested run. Skip unchanged sources. Preserve - failed observations and serialize updates to each rolling cache. -- Qualify the registered rolling paths for Immich, Mastodon, PostHog and n8n, - and add and qualify OBS rolling. Their existing diagnostic paths do not - establish the current comparison contract. -- Qualify source advancement and add rolling paths for the seven new REAPI and - Nix cases. Their successful cold/warm screens use fixed source pins. -- Preserve Zed's build-verified source promotion and Deno's adjacent-commit - sequence when moving their current source controllers. -- Run the central dispatch on GitHub, check receipts and workload completion, - and verify the selected stable release supports each scheduled case. -- Remove replaced historical cron triggers, preserving manual and release - entrypoints. Change monitoring and publication ownership with that migration. - -After a compatible stable release and qualification, remove replaced historical -cron triggers, verify there are no outstanding duplicate dispatches, then set -`BENCHMARK_CADENCE_ACTIVE=true`. Merging this branch alone does not activate the -central jobs. Passing configuration tests does not complete the rolling or -source promotion work above. +| Weekly fresh | Monday 04:00 | Latest compatible published stable release | Declared pins | +| Nightly fresh | Daily 01:17 | Latest published canary | Declared pins | +| Rolling source check | Hourly, at :11 | Stable after activation; canary for pre-activation planning | Reviewed upstream candidates | + +## Independent dispatch and ordered rolling caches + +Fresh dispatch validates the whole suite and resolves one exact CLI tag before +starting a matrix of independent requests. Each target retains its own receipt. +The combined receipt preserves requested run IDs, failed or uncertain requests, +and missing targets. A dispatch succeeding does not establish that the builds +passed. Dry runs request no builds. Interrupted requests require receipt review +before retrying; an absent HTTP response does not prove GitHub rejected a run. + +Source checks fan out by case, with six checks running at a time. Each case owns +its source-controller lock; one failed or slow case does not stop the others. +Rolling workflows serialize publication to the same cache with `queue: max`. +The next source cycle inspects the previous requested runs before advancing that +case again. An unchanged source does not request another build. + +Source inspection runs with read-only repository permissions. A separate job +reconstructs only permitted source-pin changes, verifies the expected case and +shared harness, then conditionally publishes against the current branch head. +It records dispatch intent before making requests and retains receipts under +`migration/rolling/`, including an observation per candidate source. A concurrent +edit to the same case or shared harness blocks publication; an unrelated case's +commit can be retried against its new branch head. Publication uses GitHub's +[conditional commit API](https://docs.github.com/en/graphql/reference/commits#createcommitonbranch). + +Deno retains adjacent-commit order. OBS retains its existing selection of the +next build-relevant change and its immediate parent, recording the previous +source separately. Zed selects dependency-verified candidates and promotes a +source only after its rolling build succeeds. Other cases retain their existing +promote-then-build order. Failed builds remain observations. Uncertain requests +block that case from automatic redispatch until their receipts are reviewed. + +Before activation, the publication job only plans with a canary. It does not +change `main` or request rolling builds. Both the workflow and publisher require +active central ownership on `main` before publication. + +## Workloads and reports + +Every scheduled case now registers a rolling path, including the four native +cases, OBS, five REAPI cases and two Nix cases previously missing one. Definitions +and passing local checks do not establish live rolling qualification. + +Native Docker runs load and inspect their output for both providers. REAPI +rolling runs retain the case/series cache; bazel-remote restores and publishes a +GitHub Actions cache and records the previous source and restored key. Nix and +OBS rolling runs permit both restore and publication. Missing seeds remain +bootstrap observations; they must not be described as proven cache reuse. + +Nix workers import one common dependency store prepared outside the measured +build. The export excludes the measured package, checksums the archive and +checks the imported dependency hashes against the declared derivation. This +avoids independently rebuilding a nondeterministic dependency for each arm while +keeping strict package and dependency comparisons. The earlier Zed Nix screen +restored both providers' package outputs correctly but failed the dependency +baseline comparison; that failed run remains retained. + +Shared workflows use case, variant, lane, provider and phase labels. REAPI, Nix +and OBS rolling results use the canonical benchmark report and retain both +structured evidence and the comparison summary. Reports contain measurements +and verification results, without performance verdicts. + +## Qualification and activation + +The October 5 preflight rejected stable `v1.33.0`: it does not expose +`cache-registry --reapi-port`. Canary `vcli-canary-7a5b27146ebe` passed the full +31-target preflight. Weekly activation must wait for a compatible stable release; +it must not substitute a canary or silently omit the REAPI cases. + +The first hosted source matrix completed the cases independently and identified +recipe changes in Immich, Qdrant, msgpack and Zed. Their reviewed changes are +recorded in [the recipe review](recipe-reviews-2026-10-05.md). New source pins and +changed recipes still require live qualification. + +The repository's Actions cache limit is 200 GB, with seven-day retention. +Capacity alone does not establish that a particular rolling seed was retained +or restored. + +Remaining activation requirements: + +- Pass hosted checks for the complete controller and rolling changes. +- Qualify the new and changed rolling paths with actual source advancement, + retained seeds, verified outputs and canary CLI execution. +- Pass the corrected Zed Nix cold/warm screen using the common dependency seed. +- Verify publication and receipt reconciliation in a bounded rehearsal. +- Retire replaced historical cron triggers while preserving manual and release + entrypoints; verify no outstanding duplicate dispatches. +- After a compatible stable release, change monitoring and publication ownership + and set `BENCHMARK_CADENCE_ACTIVE=true`. diff --git a/docs/recipe-reviews-2026-10-05.md b/docs/recipe-reviews-2026-10-05.md new file mode 100644 index 000000000..513424b37 --- /dev/null +++ b/docs/recipe-reviews-2026-10-05.md @@ -0,0 +1,24 @@ +# Upstream recipe review, October 5 + +The hourly source check at [run 37315116759](https://github.com/boringcache/benchmarks/actions/runs/37315116759) +identified four recipe changes. These reviews approve the recipe definitions for +qualification; they do not establish that a new-source build or cache restore passed. + +| Case | Reviewed candidate | Change and measurement impact | +| --- | --- | --- | +| Immich | `f938b2faa77cc25064c2ce54c956c40e4dd4ee79` | The Docker workflow adds a machine-learning OpenVINO runner mapping, outside the server workload. The server Dockerfile advances both pinned base images from 202609291109 to 202610021601. The server command and output check remain the same; new base images require a new build observation. | +| Qdrant | `48199a3c2d4156d0ab7e692f3925a9ed14059008` on `dev` | Integration jobs now download one shared debug binary and split consensus tests into shards. E2E jobs build one Docker image, export it to an artifact and load it on test workers. The Dockerfile and build arguments are unchanged. The benchmark retains local image loading and verification; artifact transfer and test sharding remain outside its build measurement. | +| msgpack | `ec75ffc01364adc6ea9b5f4d9129fd63602e6a45` | `project/build.properties` advances sbt 2.0.9 to 2.0.10. `build.sbt`, `project/plugins.sbt` and the launcher are unchanged. Native REAPI execution must be screened with the new sbt version. | +| Zed | `dff535449c663f3e2fde9256db2620c2ace2bc4c` | The release packaging script changes Sentry handling, linker flags and stripping of packaged copies. The rolling workload remains its declared `cargo build --release --locked`; it does not measure the release packaging script. The separately pinned release-layer plans and their recipe contract remain unchanged. | + +Each contract contains the new reviewed file hashes. Its existing parent source +can still be verified using `upstream_files_by_revision`, keyed by its complete +commit SHA. That exception retains the same reviewed file set and does not apply +to other revisions. Source advancement still requires recipe verification; a +later change to any reviewed file blocks advancement again. + +The Gogs rolling qualification also permits the exact historical parent +`103893ed5f78d2f138f5286a2dd3b4e94d18fb0b`. Its web manifest uses `@pierre/diffs` +1.2.3; the declared child advances it to 1.4.1 with matching lockfile updates. +The Moon build commands are unchanged. These are changed-source observations, +including changed dependencies, rather than identical-source replay observations. diff --git a/results/helix-nix/cadence-fresh-01/series.json b/results/helix-nix/cadence-fresh-01/series.json new file mode 100644 index 000000000..417c0d9b0 --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "helix-nix", + "series_id": "cadence-fresh-01", + "created_at": "2026-10-05T13:54:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "2dd1df2356f578abac4510efe88128471d1c69e94ff80eb775b9ba0f5ef84e95", + "variant": null, + "workflow_inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Helix package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "verification": [ + "Verify result/bin/hx exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "c2a0ba9332802d28a66f0938b4fd31a43ff928f6d8e7348f2f07f37e1c933421" +} diff --git a/results/zed-nix/cadence-fresh-01/series.json b/results/zed-nix/cadence-fresh-01/series.json new file mode 100644 index 000000000..1f6e46187 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/series.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "cadence-fresh-01", + "created_at": "2026-10-05T13:54:56Z", + "lane": "fresh", + "phases": [ + "cold", + "warm" + ], + "sample_count": 2, + "definition_sha256": "a0fb9c3b8218a12fcbe29bf77a1db64caae6b492a0291b61edc902b404b79d88", + "variant": null, + "workflow_inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed" + }, + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer. All workers import the same checksummed dependency-store export before timing; its derivation and NAR hashes must match, and the measured package is excluded from the seed.", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes. Rolling runs retain the case and series cache across commits; an initial run or evicted seed is a bootstrap observation, not proof of reuse.", + "sample_count": 2 + }, + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "plan_sha256": "97e97ded96f1686fba28581b92a015bace71e0fcaf38d7ee82235e290ab4f7cd" +} diff --git a/results/zed-nix/canary-screening-02/completions/37296214039.json b/results/zed-nix/canary-screening-02/completions/37296214039.json new file mode 100644 index 000000000..482cb44bf --- /dev/null +++ b/results/zed-nix/canary-screening-02/completions/37296214039.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" +} diff --git a/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json b/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..0b1948f1b --- /dev/null +++ b/results/zed-nix/canary-screening-02/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "canary-screening-02", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T10:22:54Z", + "ref": "run-tool-family-cases", + "state": "requested", + "run_id": 37296214039, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039" +} diff --git a/results/zed-nix/canary-screening-02/report.json b/results/zed-nix/canary-screening-02/report.json new file mode 100644 index 000000000..cfa796cd4 --- /dev/null +++ b/results/zed-nix/canary-screening-02/report.json @@ -0,0 +1,424 @@ +{ + "schema_version": 1, + "plan_sha256": "232baf4cadd9705cb853cd1410c1724fbb6b3869458cfa2bb489ea9148570a77", + "complete": false, + "valid_for_comparison": false, + "failures": [], + "methodology_issues": [], + "execution_verified": false, + "missing_completions": [], + "failed_completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" + } + ], + "completions": [ + { + "schema_version": 1, + "case_id": "zed-nix", + "series_id": "canary-screening-02", + "sample": 1, + "run_id": "37296214039", + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37296214039", + "source_sha": "8f3c254c1e080a69d08446f224be655e6bbffb36", + "github_conclusion": "failure", + "status": "failed", + "errors": [ + "Workflow concluded failure", + "cachix zed-nix warm: failure", + "boringcache zed-nix warm: failure" + ], + "evidence_manifest_sha256": "3a2c88abb871648fc13d495f49cd9476f0b6c20045d2f4828c2bf6203d60d1fd" + } + ], + "publication": "unreviewed", + "evidence_preservation": "requires-review", + "primary_metric": "build_and_reuse_seconds", + "missing": [ + [ + 1, + "warm", + "boringcache" + ], + [ + 1, + "warm", + "cachix" + ] + ], + "summaries": [ + { + "phase": "cold", + "provider": "boringcache", + "count": 1, + "measurement": { + "median": 2788.14372093361, + "min": 2788.14372093361, + "max": 2788.14372093361 + }, + "storage": null, + "storage_measured_count": 0 + }, + { + "phase": "cold", + "provider": "cachix", + "count": 1, + "measurement": { + "median": 2996.7530764946086, + "min": 2996.7530764946086, + "max": 2996.7530764946086 + }, + "storage": null, + "storage_measured_count": 0 + } + ], + "records": [ + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3193912506103516, + "build_seconds": 2785.8243296829996, + "workflow_seconds": null, + "total_seconds": 2788.14372093361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2788.14372093361 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173730", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] + }, + { + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.6468822956085205, + "build_seconds": 2994.106194199, + "workflow_seconds": null, + "total_seconds": 2996.7530764946086, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2996.7530764946086 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173731", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] + } + ], + "exclusions": [] +} diff --git a/results/zed-nix/canary-screening-02/report.md b/results/zed-nix/canary-screening-02/report.md new file mode 100644 index 000000000..1ecd33f28 --- /dev/null +++ b/results/zed-nix/canary-screening-02/report.md @@ -0,0 +1,36 @@ +# zed-nix: canary-screening-02 + +Question: Does Nix restore the Zed package closure on a fresh store? + +Measured scope: Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer + +Observations: 2/4 recorded; 0 failed; 2 missing. + +Completion checks: missing or failed. + +Publication: unreviewed. + +| Phase | Provider | Successful observations | Median build and cache reuse (s) | Range | Storage median (bytes) | Storage observations | +| --- | --- | ---: | ---: | --- | ---: | ---: | +| Cold build | BoringCache | 1 | 2788.14372093361 | 2788.14372093361–2788.14372093361 | unmeasured | 0 | +| Cold build | Cachix | 1 | 2996.7530764946086 | 2996.7530764946086–2996.7530764946086 | unmeasured | 0 | + +Run 37296214039 failed completion checks: + +- Workflow concluded failure +- cachix zed-nix warm: failure +- boringcache zed-nix warm: failure + +Missing observations: + +- Sample 1, BoringCache, Warm build +- Sample 1, Cachix, Warm build + +## Observations + +| Sample | Provider | Phase | Cache setup/restore (s) | Build (s) | Build and cache reuse (s) | Storage (bytes) | Storage source | Cache | Record | +| ---: | --- | --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| 1 | BoringCache | Cold build | 2.3193912506103516 | 2785.8243296829996 | 2788.14372093361 | unmeasured | unmeasured | miss | [JSON](runs/1-boringcache-cold.json) | +| 1 | Cachix | Cold build | 2.6468822956085205 | 2994.106194199 | 2996.7530764946086 | unmeasured | unmeasured | miss | [JSON](runs/1-cachix-cold.json) | + +[Full records and checks](report.json) diff --git a/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json b/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json new file mode 100644 index 000000000..2b7fd338c --- /dev/null +++ b/results/zed-nix/canary-screening-02/runs/1-boringcache-cold.json @@ -0,0 +1,187 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "boringcache", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.3193912506103516, + "build_seconds": 2785.8243296829996, + "workflow_seconds": null, + "total_seconds": 2788.14372093361, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2788.14372093361 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "workspace": "boringcache/benchmarks", + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": { + "workspace": "boringcache/benchmarks", + "tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "complete": false, + "measured_bytes": 0, + "unmeasured_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "total_bytes": null, + "observations": [ + { + "requested_tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "tag": "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1", + "status": "miss", + "kv_total_size": 0 + } + ] + } + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": { + "schema_version": 1, + "cli_version": "1.34.0", + "action_ref": "boringcache/one@f0fb9b2d926a32b10c543e92093ba00c5a291b79", + "action_sha": "f0fb9b2d926a32b10c543e92093ba00c5a291b79" + }, + "action": { + "resolved_mode": "nix", + "resolved_tags": [ + "zed-nix-zed-nix-series-canary-screening-02-s1-r37296214039-a1" + ], + "trust_state": { + "status": "publish", + "event_name": "workflow_dispatch", + "requested_policy": "publish", + "resolved_policy": "publish", + "write_allowed": true, + "decision_source": "cli", + "reason": "explicit_request", + "provider": "github-actions", + "detail": "Publication updates the published cache tag.", + "next_step": "Let the run finish; the next matching run restores what it publishes.", + "token_capabilities": { + "restore": true, + "stage": true, + "save": true + } + }, + "diagnostics_level": "off" + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173730", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] +} diff --git a/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json b/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json new file mode 100644 index 000000000..85046bba8 --- /dev/null +++ b/results/zed-nix/canary-screening-02/runs/1-cachix-cold.json @@ -0,0 +1,145 @@ +{ + "schema_version": 1, + "benchmark": "zed-nix", + "strategy": "cachix", + "lane": "fresh", + "phase": "cold", + "variant": null, + "mode": "nix", + "adapter": "nix", + "case": { + "schema_version": 1, + "case_id": "zed-nix", + "workspace": "boringcache/benchmarks", + "definition_sha256": "1b3c69f6a80902c353f2f99374ee450af08f423e8f14642cfe60086f59ffa237", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "origin": { + "repository": "zed-industries/zed", + "branch": "main", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "verification": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ], + "comparison": { + "question": "Does Nix restore the Zed package closure on a fresh store?", + "providers": [ + "boringcache", + "cachix" + ], + "primary_metric": "build_and_reuse_seconds", + "timed_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "storage": "provider-reported", + "cache_scope": "BoringCache case/series tag; Cachix content-addressed cache shared across series. Cold disables substitution after dependency preparation. Warm uses a fresh local output store and only the selected provider; compare exact output and dependency closure hashes.", + "sample_count": 2 + }, + "execution": { + "sync": "fixed", + "workflows": [ + { + "path": ".github/workflows/nix-fresh-benchmark.yml", + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all" + } + } + ], + "blockers": [], + "cache_tags": { + "zed-nix": "zed-nix-{scope}" + } + } + }, + "timing": { + "restore_or_setup_seconds": 2.6468822956085205, + "build_seconds": 2994.106194199, + "workflow_seconds": null, + "total_seconds": 2996.7530764946086, + "dependency_setup_seconds": null, + "compile_seconds": null, + "verification_seconds": null, + "save_seconds": null, + "comparison_scope": "Nix package derivation build or substitution after its build dependencies are realized; dependency preparation, output verification and provider post-step publication are outside the build timer", + "build_and_reuse_seconds": 2996.7530764946086 + }, + "cache": { + "hit": false, + "sccache_proof": null, + "import_ready": null, + "import_refs": 0, + "docker_plan": null, + "tag": null, + "workspace": null, + "storage_bytes": null, + "storage_source": null, + "storage_breakdown": null + }, + "source": { + "repository": "zed-industries/zed", + "sha": "96837d78cb0f2128c1965716f8df56b8aea59742" + }, + "product_refs": {}, + "action": { + "resolved_mode": null, + "resolved_tags": null, + "trust_state": null, + "diagnostics_level": null + }, + "github": { + "repository": "boringcache/benchmarks", + "run_id": "37296214039", + "run_attempt": "1", + "job": "cold", + "workflow": "Nix comparison fresh", + "ref_name": "run-tool-family-cases", + "runner_os": "Linux", + "runner_arch": "X64", + "runner_name": "GitHub Actions 1000173731", + "runner_environment": "github-hosted", + "runner_image": "ubuntu24", + "runner_image_version": "20260927.320.1" + }, + "run_uid": "gh-37296214039-1", + "series": { + "id": "canary-screening-02", + "sample": 1 + }, + "environment": { + "os": "Linux", + "arch": "X64", + "image": "ubuntu24", + "image_version": "20260927.320.1", + "machine": "ubuntu-24.04" + }, + "verification": { + "passed": true, + "checks": [ + "Output verification completed before recording this phase" + ], + "declared_checks": [ + "Verify result/bin/zed exists and its version command succeeds", + "Record the output store path and recursive closure NAR hashes in both provider arms", + "Require identical output store paths and closure hashes for the same source and flake.lock", + "On a fresh warm store, forbid local builds and require substitution from the selected provider", + "Require completed closure publication before starting the warm consumer" + ] + }, + "evidence_links": [ + "https://github.com/boringcache/benchmarks/actions/runs/37296214039" + ] +} diff --git a/schemas/case.schema.json b/schemas/case.schema.json index cb07e5bde..f602792d5 100644 --- a/schemas/case.schema.json +++ b/schemas/case.schema.json @@ -60,7 +60,8 @@ "variants": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}}} } }, - "sync": {"enum": ["upstream-head", "adjacent-pair", "verified-pair", "fixed"]}, + "sync": {"enum": ["upstream-head", "adjacent-pair", "build-relevant-pair", "verified-pair", "fixed"]}, + "rolling_workflow": {"type": "string", "pattern": "^\\.github/workflows/[A-Za-z0-9._-]+\\.ya?ml$"}, "sync_minute": {"type": "integer", "minimum": 1, "maximum": 29}, "source_prefix": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"}, "cache_tags": {"type": "object", "minProperties": 1, "additionalProperties": {"type": "string", "pattern": "^[a-z0-9._-]*\\{scope\\}[a-z0-9._-]*$"}}, diff --git a/scripts/benchmark-cadence.rb b/scripts/benchmark-cadence.rb index 5d93f4f5f..ba56e8615 100644 --- a/scripts/benchmark-cadence.rb +++ b/scripts/benchmark-cadence.rb @@ -39,15 +39,18 @@ def self.source_cases(root: BenchmarkCases::ROOT, case_id: nil) ids.select { |id| !case_id || id == case_id }.map { |id| BenchmarkCases.load_case(id, root) } end - def self.rolling_targets(root: BenchmarkCases::ROOT, case_id:) + def self.rolling_targets(root: BenchmarkCases::ROOT, case_id:, inputs: {}, item: nil) source_cases(root: root, case_id: case_id) - item = BenchmarkCases.load_case(case_id, root) + item ||= BenchmarkCases.load_case(case_id, root) workflows = item.fetch("execution").fetch("workflows").select { |workflow| workflow.fetch("lane") == "rolling" } + if (selected = item.dig("execution", "rolling_workflow")) + workflows.select! { |workflow| workflow.fetch("path") == selected } + end raise Error, "#{case_id} has no rolling workflow" if workflows.empty? cases(root: root).select { |entry| entry.fetch("case_id") == case_id }.flat_map do |entry| workflows.map do |workflow| BenchmarkCases.plan(item, lane: "rolling", workflow: File.basename(workflow.fetch("path")), - variant: entry["variant"], root: root) + variant: entry["variant"], inputs: inputs, root: root) end end.uniq end diff --git a/scripts/benchmark-cases.rb b/scripts/benchmark-cases.rb index a2bd76406..a6aa7a52d 100644 --- a/scripts/benchmark-cases.rb +++ b/scripts/benchmark-cases.rb @@ -304,6 +304,7 @@ def self.sync_source(item, root: ROOT) {"case_id" => item.fetch("id"), "updated" => true, "base_sha" => current_sha, "head_sha" => next_sha} else old_source = File.read(File.join(workload, "benchmark-source.env")) + old_settings = BenchmarkPlan.settings(File.join(workload, "benchmark-source.env")) output_path = File.join(directory, "outputs") args = ["bash", "scripts/advance-source-pair.sh", "benchmark-source.env", item.fetch("execution").fetch("source_prefix")] args << "check_dependencies" if kind == "verified-pair" @@ -326,6 +327,8 @@ def self.sync_source(item, root: ROOT) end end update_source(item, workload, root: root) if updated && kind != "verified-pair" + fields.merge!("base_sha" => values.fetch("#{prefix}_BASE_SHA"), "head_sha" => values.fetch("#{prefix}_HEAD_SHA"), + "previous_sha" => old_settings.fetch("#{prefix}_HEAD_SHA")) if updated fields.merge("case_id" => item.fetch("id"), "updated" => updated, "requires_verified_build" => kind == "verified-pair", "candidate_source_env" => updated && File.read(File.join(workload, "benchmark-source.env"))) end @@ -578,6 +581,16 @@ def self.write_contract_views(directory, root: ROOT) Dir[File.join(target, ".github", "**", "*.{yml,yaml}")].each do |path| next if path == provider_path document = YAML.safe_load(File.read(path), aliases: true) + if File.basename(path).match?(/\A(?:reapi|nix)-(?:fresh|rolling)-benchmark\.yml\z/) + inputs = (document["on"] || document[true]).dig("workflow_dispatch", "inputs") + inputs.fetch("case_id")["default"] = item.fetch("id") + providers = inputs.fetch("provider").fetch("options") - ["all"] + raise Error, "Provider selector differs from the declared comparison" unless providers.sort == item.dig("comparison", "providers").sort + document.fetch("jobs").each_value do |job| + matrix = job.dig("strategy", "matrix") + matrix["provider"] = providers if matrix && matrix["provider"].is_a?(String) + end + end if item.dig("execution", "native") && File.basename(path).start_with?("native-") recipe = NativeCase.resolve(item.dig("execution", "native"), native_variant) document["env"]["BENCHMARK_ID"] = "${{ format('#{recipe.fetch('benchmark_id')}{0}', inputs.benchmark_id_suffix) }}" diff --git a/scripts/benchmark-phase.rb b/scripts/benchmark-phase.rb index 2e48586b1..a47955316 100644 --- a/scripts/benchmark-phase.rb +++ b/scripts/benchmark-phase.rb @@ -8,6 +8,11 @@ def self.scope(env, require_published: false) lane = env.fetch("CACHE_LANE") raise "Use fresh or rolling" unless %w[fresh rolling].include?(lane) raise "Invalid benchmark ID" unless id.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) + rolling_scope = env.fetch("BENCHMARK_ROLLING_SCOPE", "") + if lane == "rolling" && !rolling_scope.empty? + raise "Invalid rolling cache scope" unless rolling_scope.match?(/\A[a-z0-9][a-z0-9._-]+\z/) + return "#{id}-rolling-#{rolling_scope}" + end series = env.fetch("BENCHMARK_SERIES_ID", "") if !series.empty? raise "Invalid series ID" unless series.match?(/\A[a-z0-9]+(?:-[a-z0-9]+)*\z/) diff --git a/scripts/check-report-contract.rb b/scripts/check-report-contract.rb index 703c4f296..92b5c258e 100755 --- a/scripts/check-report-contract.rb +++ b/scripts/check-report-contract.rb @@ -209,7 +209,13 @@ def producers_for(workflow:, repo_dir:, job_id:, job:) next unless step.is_a?(Hash) uses = step["uses"].to_s - if uses.start_with?("./.github/actions/") + if %w[./.github/actions/reapi-benchmark ./.github/actions/nix-benchmark].include?(uses) + with = step.fetch("with") + fields = {"benchmark" => workflow.resolve(with.fetch("case-id"), matrix), + "strategy" => workflow.resolve(with.fetch("provider"), matrix), + "lane" => workflow.resolve(with.fetch("lane", "fresh"), matrix), + "phase" => workflow.resolve(with.fetch("phase"), matrix), "variant" => ""} + elsif uses.start_with?("./.github/actions/") with = step["with"].is_a?(Hash) ? step["with"] : {} variant_input = action_variant_input(repo_dir, uses, with) variant = workflow.resolve(with[variant_input].to_s, matrix).to_s if variant_input @@ -220,6 +226,12 @@ def producers_for(workflow:, repo_dir:, job_id:, job:) "phase" => workflow.resolve(with["phase"].to_s, matrix), "variant" => variant_input && variant != DEFAULT_PLATFORM ? variant : "" } + elsif step["run"].to_s.include?("write_phase_result.rb") + flags = step.fetch("run").scan(/--(surface|strategy|phase)\s+([^\s\\]+)/).to_h + surface = flags.fetch("surface") + base = flags.fetch("phase") == "base" + fields = {"benchmark" => step.fetch("run").include?("--continuous") ? "obs-studio" : "obs-studio-#{surface}", + "strategy" => flags.fetch("strategy"), "lane" => base ? "fresh" : "rolling", "phase" => base ? "cold" : "commit", "variant" => surface} elsif step["run"].to_s.match?(/benchmark-report\.rb\s+phase\b/) shell_env = workflow.shell_env_for(job, step, matrix) fields = phase_flags(step["run"].to_s).transform_values { |value| workflow.resolve(value, matrix, shell_env) } diff --git a/scripts/nix-benchmark.rb b/scripts/nix-benchmark.rb index 8dfaa0bca..d22ddfea1 100644 --- a/scripts/nix-benchmark.rb +++ b/scripts/nix-benchmark.rb @@ -4,6 +4,7 @@ require "open3" require "fileutils" require "uri" +require "digest" require_relative "benchmark-plan" module NixBenchmark @@ -35,7 +36,7 @@ def self.closure(paths) end.to_h end - def self.prepare + def self.prepare(seed: nil) recipe = BenchmarkPlan.load.fetch("adapters").fetch("nix") command = recipe.fetch("command") reference = command.fetch(2) @@ -48,7 +49,34 @@ def self.prepare realized = capture("nix-store", "--realise", *dependencies).lines.map(&:strip) outputs = capture("nix-store", "--query", "--outputs", drv).lines.map(&:strip) raise Error, "Package output already exists before measurement" if outputs.any? { |path| File.exist?(path) } - write("nix-evidence/baseline.json", {"derivation" => drv, "outputs" => outputs, "dependencies" => closure(realized), "nix_version" => capture("nix", "--version")}) + baseline = {"derivation" => drv, "outputs" => outputs, "dependencies" => closure(realized), "nix_version" => capture("nix", "--version")} + if seed + expected = JSON.parse(File.read(File.join(seed, "baseline.json"))) + raise Error, "Dependency seed differs from the declared workload" unless baseline == expected + end + write("nix-evidence/baseline.json", baseline) + end + + def self.export_dependencies(directory: "nix-dependencies") + baseline = JSON.parse(File.read("nix-evidence/baseline.json")) + paths = baseline.fetch("dependencies").keys + raise Error, "Dependency export includes the measured package" unless (paths & baseline.fetch("outputs")).empty? + raise Error, "No dependency store paths to export" if paths.empty? + FileUtils.mkdir_p(directory) + archive = File.join(directory, "store.nar.zst") + statuses = Open3.pipeline(["nix-store", "--export", *paths], ["zstd", "-T0", "-3", "-o", archive]) + raise Error, "Dependency export failed" unless statuses.all?(&:success?) + write(File.join(directory, "baseline.json"), baseline) + write(File.join(directory, "archive.json"), {"sha256" => Digest::SHA256.file(archive).hexdigest}) + end + + def self.import_dependencies(directory:) + archive = File.join(directory, "store.nar.zst") + expected = JSON.parse(File.read(File.join(directory, "archive.json"))).fetch("sha256") + raise Error, "Dependency archive checksum differs" unless Digest::SHA256.file(archive).hexdigest == expected + statuses = Open3.pipeline(["zstd", "-dc", archive], ["nix-store", "--import"]) + raise Error, "Dependency import failed" unless statuses.all?(&:success?) + prepare(seed: directory) end def self.provider_substituter(config, provider, cache_name) @@ -68,18 +96,24 @@ def self.provider_substituter(config, provider, cache_name) matches.first end - def self.build(phase:, provider:, cache_name: "") - raise Error, "Use cold or warm" unless %w[cold warm].include?(phase) - baseline = JSON.parse(File.read("nix-evidence/baseline.json")) - raise Error, "Package output already exists before measurement" if baseline.fetch("outputs").any? { |path| File.exist?(path) } - command = BenchmarkPlan.command("nix") + def self.build_command(phase:, provider:, cache_name: "") + raise Error, "Use cold, warm or commit" unless %w[cold warm commit].include?(phase) + command = BenchmarkPlan.command("nix").dup if phase == "cold" command += %w[--option substitute false] else config = JSON.parse(capture("nix", "config", "show", "--json")) url = provider_substituter(config, provider, cache_name) - command += ["--max-jobs", "0", "--builders", "", "--option", "substituters", url, "--option", "extra-substituters", ""] + command += ["--option", "substituters", url, "--option", "extra-substituters", ""] + command += ["--max-jobs", "0", "--builders", ""] if phase == "warm" end + command + end + + def self.build(phase:, provider:, cache_name: "") + baseline = JSON.parse(File.read("nix-evidence/baseline.json")) + raise Error, "Package output already exists before measurement" if baseline.fetch("outputs").any? { |path| File.exist?(path) } + command = build_command(phase: phase, provider: provider, cache_name: cache_name) started = Process.clock_gettime(Process::CLOCK_MONOTONIC) success = false File.open("nix-evidence/build.log", "w") do |log| @@ -125,6 +159,8 @@ def self.compare(seed:, current: "nix-evidence") begin case ARGV.shift when "prepare" then NixBenchmark.prepare + when "export-dependencies" then NixBenchmark.export_dependencies + when "import-dependencies" then NixBenchmark.import_dependencies(directory: ARGV.fetch(0)) when "build" then NixBenchmark.build(phase: ENV.fetch("NIX_PHASE"), provider: ENV.fetch("NIX_PROVIDER"), cache_name: ENV.fetch("CACHIX_CACHE", "")) when "verify" then NixBenchmark.verify(phase: ENV.fetch("NIX_PHASE"), provider: ENV.fetch("NIX_PROVIDER"), cache_name: ENV.fetch("CACHIX_CACHE", "")) when "compare" then NixBenchmark.compare(seed: ARGV.fetch(0)) diff --git a/scripts/reapi-registry.rb b/scripts/reapi-registry.rb index 88b460b4f..2a8a4230c 100644 --- a/scripts/reapi-registry.rb +++ b/scripts/reapi-registry.rb @@ -60,7 +60,7 @@ def self.with_process(command, port:, log:, startup_timeout: 120, shutdown_timeo end def self.command(provider:, phase:, workspace:, tag:, port: 5060) - raise Error, "Use cold or warm" unless %w[cold warm].include?(phase) + raise Error, "Use cold, warm or commit" unless %w[cold warm commit].include?(phase) case provider when "boringcache" args = ["boringcache", "cache-registry", workspace, tag, "--port", "5061", "--reapi-port", port.to_s, diff --git a/scripts/reapi-setup.rb b/scripts/reapi-setup.rb index 09875ecf0..86d65b38d 100644 --- a/scripts/reapi-setup.rb +++ b/scripts/reapi-setup.rb @@ -61,12 +61,31 @@ def self.prepare ReapiClient.recipe.fetch("prepare").each { |command| run(*command) } end + def self.rolling_seed + context = JSON.parse(File.read("benchmark-context.json")) + path = "reapi-store/benchmark-source.json" + previous = File.file?(path) ? JSON.parse(File.read(path)) : nil + if previous && previous.fetch("case_id") != ENV.fetch("BENCHMARK_ID") + raise "Rolling comparator seed belongs to another case" + end + seed = {"cache_key" => ENV.fetch("RESTORED_CACHE_KEY", ""), "previous" => previous, + "current" => {"case_id" => ENV.fetch("BENCHMARK_ID"), "source" => context.fetch("source"), + "run_id" => ENV.fetch("GITHUB_RUN_ID"), "run_attempt" => ENV.fetch("GITHUB_RUN_ATTEMPT")}} + File.write("reapi-evidence/rolling-seed.json", JSON.pretty_generate(seed) + "\n") + end + + def self.publish_seed + seed = JSON.parse(File.read("reapi-evidence/rolling-seed.json")) + FileUtils.mkdir_p("reapi-store") + File.write("reapi-store/benchmark-source.json", JSON.pretty_generate(seed.fetch("current")) + "\n") + end + def self.report require_relative "benchmark-report" context = JSON.parse(File.read("benchmark-context.json")) timings = JSON.parse(File.read("reapi-evidence/timing.json")) args = ["phase", "--verified-output", "--benchmark", ENV.fetch("BENCHMARK_ID"), "--strategy", ENV.fetch("PROVIDER"), - "--lane", "fresh", "--phase", ENV.fetch("PHASE"), "--mode", "reapi", + "--lane", ENV.fetch("CACHE_LANE", "fresh"), "--phase", ENV.fetch("PHASE"), "--mode", "reapi", "--workspace", "boringcache/benchmarks", "--cache-tag", BenchmarkPhase.scope(ENV), "--source-repository", context.dig("source", "repository"), "--source-sha", context.dig("source", "revision"), "--build-seconds", timings.fetch("build_seconds").to_s, "--restore-or-setup-seconds", timings.fetch("restore_or_setup_seconds").to_s, @@ -87,6 +106,8 @@ def self.report when "install" then ReapiSetup.install when "prepare" then ReapiSetup.prepare when "report" then ReapiSetup.report + when "rolling-seed" then ReapiSetup.rolling_seed + when "publish-seed" then ReapiSetup.publish_seed else abort "Use install, prepare or report" end end diff --git a/scripts/source-promotion.rb b/scripts/source-promotion.rb new file mode 100644 index 000000000..1c5eb0b01 --- /dev/null +++ b/scripts/source-promotion.rb @@ -0,0 +1,291 @@ +# frozen_string_literal: true + +require "base64" +require_relative "benchmark-cadence" + +# Publication uses reviewed harness code only. Candidate workloads run in jobs +# with read-only repository access; this module never executes their code. +module SourcePromotion + class Error < StandardError; end + + def self.changes(item, proposal, root: BenchmarkCases::ROOT) + raise Error, "Expected a changed source proposal for this case" unless proposal["state"] == "proposal" && proposal["case_id"] == item.fetch("id") + if !!proposal["requires_verified_build"] != (item.dig("execution", "sync") == "verified-pair") + raise Error, "Source proposal changes the case's build-before-promotion requirement" + end + base, head = proposal.values_at("base_sha", "head_sha") + raise Error, "Source promotion requires distinct exact revisions" unless [base, head].all? { |sha| sha.is_a?(String) && sha.match?(/\A[0-9a-f]{40}\z/) } && base != head + candidate = Marshal.load(Marshal.dump(item)) + prefix = item.dig("execution", "source_prefix") + id = item.fetch("id") + paths = {} + if prefix + path = "cases/#{id}/payload/benchmark-source.env" + original = File.read(File.join(root, path)) + settings = BenchmarkPlan.settings(File.join(root, path)) + previous = proposal.fetch("previous_sha", base) + raise Error, "Candidate does not follow the declared parent" unless settings.fetch("#{prefix}_HEAD_SHA") == previous + raise Error, "An adjacent sequence must retain the previous head as its base" if item.dig("execution", "sync") == "adjacent-pair" && base != previous + expected = original.sub(/^#{prefix}_BASE_SHA=.*$/, "#{prefix}_BASE_SHA=#{base}").sub(/^#{prefix}_HEAD_SHA=.*$/, "#{prefix}_HEAD_SHA=#{head}") + raise Error, "Source proposal changes settings other than the source pair" unless proposal.fetch("candidate_source_env") == expected + paths[path] = expected + candidate.fetch("source")["pins"] = candidate.dig("source", "pins").reject { |pin| pin["kind"] == "env" && pin["path"] == "payload/benchmark-source.env" } + + [base, head].map { |sha| {"path" => "payload/benchmark-source.env", "revision" => sha, "kind" => "env"} } + candidate.dig("source", "pins").select { |pin| pin["kind"] == "gitlink" }.each { |pin| pin["revision"] = head } + elsif item.dig("source", "revision") + raise Error, "Candidate does not follow the declared snapshot" unless item.dig("source", "revision") == base + candidate.fetch("source")["revision"] = head + candidate.dig("source", "pins").each do |pin| + raise Error, "Snapshot contains an independently pinned source" unless pin.fetch("revision") == base + pin["revision"] = head + end + reference = "github:#{item.dig('source', 'repository')}/#{base}#default" + replacement = "github:#{item.dig('source', 'repository')}/#{head}#default" + %w[.boringcache.toml recipe-contract.json].each do |name| + path = "cases/#{id}/payload/#{name}" + next unless File.file?(File.join(root, path)) + original = File.read(File.join(root, path)) + paths[path] = original.gsub(reference, replacement) if original.include?(reference) + end + else + pins = candidate.dig("source", "pins").select { |pin| pin["kind"] == "gitlink" } + raise Error, "Candidate does not follow the declared gitlink" if pins.empty? || pins.any? { |pin| pin.fetch("revision") != base } + pins.each { |pin| pin["revision"] = head } + end + paths["cases/#{id}/case.json"] = JSON.pretty_generate(candidate) + "\n" + [candidate, paths] + end + + def self.targets(item, proposal, version:, root: BenchmarkCases::ROOT) + candidate, = changes(item, proposal, root: root) + inputs = {"cli_version" => version} + case item.dig("execution", "sync") + when "adjacent-pair", "build-relevant-pair" + inputs.merge!("base_sha" => proposal.fetch("base_sha"), "head_sha" => proposal.fetch("head_sha"), "cache_scope" => "rolling-main") + when "verified-pair" + inputs.merge!("base_sha" => proposal.fetch("base_sha"), "head_sha" => proposal.fetch("head_sha"), "source_distance" => proposal.fetch("source_distance").to_s) + end + BenchmarkCadence.rolling_targets(case_id: item.fetch("id"), item: candidate, inputs: inputs, root: root) + end + + class Publisher + attr_reader :root, :repository, :branch + + def initialize(root: BenchmarkCases::ROOT, runner: NightlyCanaries::Runner.new, branch: "main") + raise Error, "Use main or an isolated cadence rehearsal branch" unless branch == "main" || branch.match?(/\Acadence-rehearsal-[a-z0-9-]+\z/) + @root, @runner = root, runner + @branch = branch + @repository = BenchmarkCases::REPOSITORY + end + + def api(path, body: nil) + if path == "graphql" + output, error, status = Open3.capture3("gh", "api", "graphql", "--method", "POST", "--input", "-", stdin_data: JSON.generate(body)) + result = JSON.parse(output) unless output.empty? + return result if result && (status.success? || result.key?("errors")) + raise Error, "GitHub publication request failed: #{error.strip}" + end + @runner.api(path, body: body) + end + + def file(path, ref: branch) + result = api("repos/#{repository}/contents/#{path}?ref=#{CGI.escape(ref)}") + raise Error, "Expected a repository file" unless result["type"] == "file" + Base64.decode64(result.fetch("content")) + end + + def optional_file(path, ref: branch) + file(path, ref: ref) + rescue NightlyCanaries::Error => error + raise unless error.message.include?("HTTP 404") + nil + end + + # Every update is conditional on both the branch head and the previous + # contents of the files this case owns. Another case may advance meanwhile. + def commit(changes, expected:, message:) + 20.times do + head = api("repos/#{repository}/git/ref/heads/#{branch}").dig("object", "sha") + protect_harness(head, except: expected.keys) if @case_id + expected.each do |path, contents| + raise Error, "#{path} changed while this source was being checked" unless optional_file(path, ref: head) == contents + end + input = {"branch" => {"repositoryNameWithOwner" => repository, "branchName" => branch}, + "expectedHeadOid" => head, "message" => {"headline" => message}, + "fileChanges" => {"additions" => changes.map { |path, contents| {"path" => path, "contents" => Base64.strict_encode64(contents)} }}} + result = api("graphql", body: {"query" => 'mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { oid } } }', "variables" => {"input" => input}}) + oid = result.dig("data", "createCommitOnBranch", "commit", "oid") + if oid&.match?(/\A[0-9a-f]{40}\z/) + @baseline_ref = oid + return oid + end + errors = result.fetch("errors", []) + # Only an optimistic-concurrency conflict is safe to retry. Other API + # failures may have an uncertain outcome and need their receipt inspected. + raise Error, "Source publication failed: #{errors.map { |error| error['message'] }.join('; ')}" unless errors.any? { |error| error["type"] == "STALE_DATA" } + end + raise Error, "Source publication could not acquire the current branch head" + end + + def protect_harness(head, except:) + @baseline_ref ||= BenchmarkCases.command("git", "rev-parse", "HEAD", chdir: root).strip + snapshots = [@baseline_ref, head].map do |ref| + tree = api("repos/#{repository}/git/trees/#{ref}?recursive=1") + raise Error, "Cannot verify a truncated harness tree" if tree["truncated"] + tree.fetch("tree").select do |entry| + path = entry.fetch("path") + entry["type"] == "blob" && !except.include?(path) && + (path.start_with?(".github/", "scripts/", "bin/", "cases/#{@case_id}/") || %w[Gemfile Gemfile.lock .tool-versions suites/scheduled.json].include?(path)) + end.to_h { |entry| [entry.fetch("path"), entry.fetch("sha")] } + end + raise Error, "The case or shared harness changed during source inspection" unless snapshots.first == snapshots.last + end + + def state_path(id) + "migration/rolling/#{id}.json" + end + + def save(record, expected:, changes: {}) + contents = JSON.pretty_generate(record) + "\n" + path = state_path(record.fetch("case_id")) + history = "migration/rolling/#{record.fetch('case_id')}/#{record.fetch('proposal').fetch('head_sha')}.json" + commit(changes.merge(path => contents, history => contents), expected: expected, + message: "Record #{record.fetch('case_id')} rolling #{record.fetch('state')}") + contents + end + + def reconcile(item, output:) + @case_id = item.fetch("id") + path = state_path(item.fetch("id")) + previous = optional_file(path) + return nil unless previous + record = JSON.parse(previous) + raise Error, "Rolling receipt belongs to another case" unless record.fetch("case_id") == item.fetch("id") + if %w[dispatching dispatch-failed].include?(record.fetch("state")) + BenchmarkCases.write_json(output, record) + raise Error, "Inspect the retained dispatch before advancing this case; a request may have been accepted" + end + return record unless record["state"] == "requested" + if record.fetch("runs").empty? || record.fetch("runs").any? { |run| !run["id"].is_a?(Integer) || !run["id"].positive? } + raise Error, "A requested rolling receipt must retain every GitHub run ID" + end + record.fetch("runs").each do |run| + current = api("repos/#{repository}/actions/runs/#{run.fetch('id')}") + run["outcome"] = current["status"] == "completed" ? current.fetch("conclusion") : current.fetch("status") + end + if record.fetch("runs").any? { |run| %w[queued requested pending waiting in_progress].include?(run["outcome"]) } + BenchmarkCases.write_json(output, record) + return record + end + success = record.fetch("runs").all? { |run| run["outcome"] == "success" } + record["state"] = success ? "completed" : "failed" + changes = {} + expected = {path => previous} + if success && record.fetch("proposal")["requires_verified_build"] + _, changes = SourcePromotion.changes(item, record.fetch("proposal"), root: root) + changes.each_key { |name| expected[name] = File.read(File.join(root, name)) } + record["state"] = "promoted" + end + save(record, expected: expected, changes: changes) + BenchmarkCases.write_json(output, record) + record + end + + def publish(item, proposal, version:, output:, dry_run: false) + @case_id = item.fetch("id") + candidate, changes = SourcePromotion.changes(item, proposal, root: root) + plans = SourcePromotion.targets(item, proposal, version: version, root: root) + BenchmarkCadence.verify_cli(version, plans) + path = state_path(item.fetch("id")) + previous = optional_file(path) + if previous + prior = JSON.parse(previous) + raise Error, "This case has an unfinished or uncertain rolling request" if %w[dispatching dispatch-failed requested].include?(prior.fetch("state")) + raise Error, "This source already has a retained rolling observation" if prior.dig("proposal", "head_sha") == proposal.fetch("head_sha") + end + runs = plans.map { |plan| plan.slice("repository", "workflow", "inputs").merge("state" => "requesting") } + record = {"schema_version" => 1, "case_id" => item.fetch("id"), "state" => "dispatching", "cli_version" => version, "ref" => branch, + "proposal" => proposal, "created_at" => Time.now.utc.iso8601, "runs" => runs} + if dry_run + record["state"] = "planned" + runs.each { |run| run["state"] = "planned" } + BenchmarkCases.write_json(output, record) + return record + end + # Zed advances only after the build succeeds. The other controllers retain + # their historical promote-then-build ordering. + changes = {} if proposal["requires_verified_build"] + expected = {path => previous, "cases/#{item.fetch('id')}/case.json" => File.read(File.join(root, "cases", item.fetch("id"), "case.json"))} + changes.each_key { |name| expected[name] = File.read(File.join(root, name)) } + BenchmarkCases.write_json(output, record) + pending = save(record, expected: expected, changes: changes) + runs.each do |run| + begin + result = api("repos/#{repository}/actions/workflows/#{run.fetch('workflow')}/dispatches", + body: {"ref" => branch, "inputs" => run.fetch("inputs"), "return_run_details" => true}) + id = result["workflow_run_id"] + raise Error, "Dispatch returned no run ID" unless id.is_a?(Integer) && id.positive? + run.merge!("id" => id, "url" => "https://github.com/#{repository}/actions/runs/#{id}", "state" => "requested") + rescue StandardError => error + run.merge!("state" => "request-unknown", "error" => error.message) + ensure + BenchmarkCases.write_json(output, record) + end + end + record["state"] = runs.all? { |run| run["state"] == "requested" } ? "requested" : "dispatch-failed" + save(record, expected: {path => pending}) + BenchmarkCases.write_json(output, record) + raise Error, "At least one rolling request has an uncertain outcome; inspect its receipt before retrying" unless record["state"] == "requested" + record + end + end +end + +if $PROGRAM_NAME == __FILE__ + options = {channel: "stable"} + OptionParser.new do |parser| + parser.on("--case ID") { |value| options[:case_id] = value } + parser.on("--proposal PATH") { |value| options[:proposal] = value } + parser.on("--output PATH") { |value| options[:output] = value } + parser.on("--channel CHANNEL") { |value| options[:channel] = value } + parser.on("--version TAG") { |value| options[:version] = value } + parser.on("--publish") { options[:publish] = true } + parser.on("--reconcile") { options[:reconcile] = true } + end.parse! + begin + item = BenchmarkCadence.source_cases(case_id: options.fetch(:case_id)).fetch(0) + if options[:publish] || options[:reconcile] + unless ENV["BENCHMARK_CADENCE_ACTIVE"] == "true" && ENV["GITHUB_REF_NAME"] == "main" && ENV["GITHUB_REPOSITORY"] == BenchmarkCases::REPOSITORY + raise SourcePromotion::Error, "Source publication requires active central ownership on main" + end + end + publisher = SourcePromotion::Publisher.new + output = options.fetch(:output) + if options[:reconcile] + record = publisher.reconcile(item, output: output) + proceed = !record || !%w[requested dispatching dispatch-failed].include?(record.fetch("state")) + if ENV["GITHUB_OUTPUT"] + head = publisher.api("repos/#{BenchmarkCases::REPOSITORY}/git/ref/heads/main").dig("object", "sha") + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") { |file| file.puts("proceed=#{proceed}\nref=#{head}") } + end + else + inventory = JSON.parse(File.read(options.fetch(:proposal))) + proposal = inventory.fetch("records").find { |record| record.fetch("case_id") == item.fetch("id") } + raise SourcePromotion::Error, "No changed source proposal" unless proposal && proposal["state"] == "proposal" + runner = NightlyCanaries::Runner.new + raise SourcePromotion::Error, "Use stable or canary" unless %w[stable canary].include?(options[:channel]) + version = if options[:version] + release = runner.api("repos/boringcache/cli/releases/tags/#{options.fetch(:version)}") + options[:channel] == "stable" ? runner.published_stable(release) : runner.published_canary(release) + else + options[:channel] == "stable" ? runner.latest_stable : runner.latest_canary + end + publisher.publish(item, proposal, version: version, output: output, dry_run: !options[:publish]) + end + rescue SourcePromotion::Error, BenchmarkCases::Error, BenchmarkCadence::Error, NightlyCanaries::Error, KeyError, JSON::ParserError => error + if options[:output] && !File.exist?(options[:output]) + BenchmarkCases.write_json(options[:output], {"case_id" => options[:case_id], "state" => "blocked", "error" => error.message}) + end + abort error.message + end +end diff --git a/scripts/verify-upstream-recipe.rb b/scripts/verify-upstream-recipe.rb index 06c276de8..80e20ee9b 100755 --- a/scripts/verify-upstream-recipe.rb +++ b/scripts/verify-upstream-recipe.rb @@ -3,11 +3,21 @@ require_relative "benchmark-plan" require "digest" +require "open3" root = BenchmarkPlan::ROOT source = ARGV.first || File.join(root, "upstream") contract = JSON.parse(File.read(File.join(root, ARGV[1] || "recipe-contract.json"))) -contract.fetch("upstream_files").each do |path, digest| +files = contract.fetch("upstream_files") +if contract.key?("upstream_files_by_revision") + revision, status = Open3.capture2("git", "-C", source, "rev-parse", "HEAD") + raise "Cannot determine the source revision for recipe verification" unless status.success? + overrides = contract.fetch("upstream_files_by_revision") + raise "Recipe exceptions require exact source revisions" unless overrides.keys.all? { |sha| sha.match?(/\A[0-9a-f]{40}\z/) } + files = overrides.fetch(revision.strip, files) + raise "Recipe exception changes the reviewed file set" unless files.keys.sort == contract.fetch("upstream_files").keys.sort +end +files.each do |path, digest| actual = File.join(source, path) raise "Upstream recipe file is missing: #{path}" unless File.file?(actual) raise "Upstream recipe changed: #{path}; review the workload before updating its recipe digest" unless Digest::SHA256.file(actual).hexdigest == digest diff --git a/test/benchmark_cadence_test.rb b/test/benchmark_cadence_test.rb index 5035b7fa6..e4bf648ec 100644 --- a/test/benchmark_cadence_test.rb +++ b/test/benchmark_cadence_test.rb @@ -115,7 +115,12 @@ def test_native_rolling_paths_verify_outputs_for_every_scheduled_variant assert targets.all? { |target| target.fetch("workflow") == "native-rolling-benchmark.yml" } assert_equal selections.map { |entry| entry["variant"] }, targets.map { |target| target.dig("inputs", "variant") } end - assert_raises(BenchmarkCadence::Error) { BenchmarkCadence.rolling_targets(case_id: "helix-nix") } + %w[gogs-moon opencut-moon stackstorm-pants executorch-buck2 msgpack-sbt helix-nix zed-nix].each do |id| + targets = BenchmarkCadence.rolling_targets(case_id: id) + assert_equal 1, targets.length + assert_equal id, targets.first.dig("inputs", "case_id") + assert_equal "rolling", targets.first.fetch("lane") + end end def test_schedules_require_explicit_cutover_and_preserve_historical_monitoring diff --git a/test/benchmark_case_scripts_test.rb b/test/benchmark_case_scripts_test.rb index 6d264525d..4802f38ea 100644 --- a/test/benchmark_case_scripts_test.rb +++ b/test/benchmark_case_scripts_test.rb @@ -338,6 +338,41 @@ def test_selected_recipe_contract_checks_each_declared_plan end end + def test_parent_recipe_exception_is_limited_to_its_exact_revision_and_file_set + with_case("hugo-go") do |directory| + source = File.join(directory, "upstream") + FileUtils.mkdir_p(source) + git = ->(*args) { Open3.capture2e("git", "-C", source, "-c", "user.name=Benchmark test", "-c", "user.email=benchmark@localhost", "-c", "commit.gpgsign=false", *args) } + _, status = git.call("init") + assert status.success? + File.write(File.join(source, "recipe.txt"), "parent recipe\n") + git.call("add", ".") + _, status = git.call("commit", "-m", "Parent recipe") + assert status.success? + revision, = git.call("rev-parse", "HEAD") + parent = {"recipe.txt" => Digest::SHA256.hexdigest("parent recipe\n")} + contract = {"upstream_files" => {"recipe.txt" => Digest::SHA256.hexdigest("new recipe\n")}, + "upstream_files_by_revision" => {revision.strip => parent}, "commands" => {}} + path = File.join(directory, "recipe-contract.json") + File.write(path, JSON.generate(contract)) + _, error, status = run_script(directory, "verify-upstream-recipe") + assert status.success?, error + git.call("commit", "--allow-empty", "-m", "Another revision") + _, error, status = run_script(directory, "verify-upstream-recipe") + refute status.success? + assert_includes error, "Upstream recipe changed" + File.write(File.join(source, "recipe.txt"), "new recipe\n") + _, error, status = run_script(directory, "verify-upstream-recipe") + assert status.success?, error + current, = git.call("rev-parse", "HEAD") + contract["upstream_files_by_revision"] = {current.strip => {}} + File.write(path, JSON.generate(contract)) + _, error, status = run_script(directory, "verify-upstream-recipe") + refute status.success? + assert_includes error, "changes the reviewed file set" + end + end + def test_deno_profile_selects_exactly_one_declared_profile with_case("deno") do |directory| path = File.join(directory, ".boringcache.toml") diff --git a/test/benchmark_phase_test.rb b/test/benchmark_phase_test.rb index aff806cce..5448f1a35 100644 --- a/test/benchmark_phase_test.rb +++ b/test/benchmark_phase_test.rb @@ -60,4 +60,14 @@ def test_warm_consumption_never_exports_and_explicit_publication_applies_to_both refute_includes File.read(path), "docker/build-push-action@", "#{path}: Docker provider lifecycle must be shared" end end + + def test_explicit_rolling_cohort_links_frozen_source_generations_without_changing_fresh_identity + env = {"BENCHMARK_ID" => "posthog", "CACHE_LANE" => "rolling", "BENCHMARK_SERIES_ID" => "parent-01", + "BENCHMARK_SAMPLE" => "1", "BENCHMARK_ROLLING_SCOPE" => "qualification-01", "GITHUB_RUN_ID" => "123", "GITHUB_RUN_ATTEMPT" => "1"} + assert_equal "posthog-rolling-qualification-01", BenchmarkPhase.scope(env) + assert_equal BenchmarkPhase.scope(env), BenchmarkPhase.scope(env.merge("BENCHMARK_SERIES_ID" => "child-01", "GITHUB_RUN_ID" => "456")) + fresh = env.merge("CACHE_LANE" => "fresh") + assert_equal BenchmarkPhase.scope(fresh.reject { |key, _| key == "BENCHMARK_ROLLING_SCOPE" }), BenchmarkPhase.scope(fresh) + assert_raises(RuntimeError) { BenchmarkPhase.scope(env.merge("BENCHMARK_ROLLING_SCOPE" => "../other")) } + end end diff --git a/test/nix_benchmark_test.rb b/test/nix_benchmark_test.rb index c00b8146c..68fc8325c 100644 --- a/test/nix_benchmark_test.rb +++ b/test/nix_benchmark_test.rb @@ -18,6 +18,22 @@ def test_ambiguous_provider_configuration_fails assert_raises(NixBenchmark::Error) { NixBenchmark.provider_substituter(config, "boringcache", "") } end + def test_changed_source_can_build_after_a_provider_miss_but_warm_cannot + config = {"substituters" => {"value" => ["https://cache.nixos.org", "https://benchmark.cachix.org"]}} + BenchmarkPlan.stub(:command, %w[nix build pinned-source]) do + NixBenchmark.stub(:capture, JSON.generate(config)) do + rolling = NixBenchmark.build_command(phase: "commit", provider: "cachix", cache_name: "benchmark") + warm = NixBenchmark.build_command(phase: "warm", provider: "cachix", cache_name: "benchmark") + assert_includes rolling, "https://benchmark.cachix.org" + refute_includes rolling, "https://cache.nixos.org" + refute_includes rolling, "--max-jobs" + assert_equal rolling + ["--max-jobs", "0", "--builders", ""], warm + assert_equal %w[nix build pinned-source --option substitute false], NixBenchmark.build_command(phase: "cold", provider: "cachix") + assert_raises(NixBenchmark::Error) { NixBenchmark.build_command(phase: "invalid", provider: "cachix") } + end + end + end + def test_preexisting_package_output_cannot_be_reported_as_a_warm_restore Dir.mktmpdir do |directory| Dir.chdir(directory) do @@ -47,4 +63,26 @@ def test_comparison_rejects_changed_dependency_and_output_hashes assert_raises(NixBenchmark::Error) { NixBenchmark.compare(seed: seed, current: current) } end end + + def test_dependency_archive_is_verified_before_import + Dir.mktmpdir do |directory| + File.write(File.join(directory, "store.nar.zst"), "changed") + NixBenchmark.write(File.join(directory, "archive.json"), {"sha256" => Digest::SHA256.hexdigest("expected")}) + Open3.stub(:pipeline, ->(*) { flunk "Unverified archive was imported" }) do + error = assert_raises(NixBenchmark::Error) { NixBenchmark.import_dependencies(directory: directory) } + assert_includes error.message, "checksum differs" + end + end + end + + def test_dependency_export_cannot_seed_the_measured_package + Dir.mktmpdir do |directory| + Dir.chdir(directory) do + NixBenchmark.write("nix-evidence/baseline.json", {"outputs" => ["/nix/store/package"], "dependencies" => {"/nix/store/package" => "hash"}}) + Open3.stub(:pipeline, ->(*) { flunk "Measured package was exported as a dependency" }) do + assert_raises(NixBenchmark::Error) { NixBenchmark.export_dependencies } + end + end + end + end end diff --git a/test/reapi_registry_test.rb b/test/reapi_registry_test.rb index cb8ebc592..09efc5bf7 100644 --- a/test/reapi_registry_test.rb +++ b/test/reapi_registry_test.rb @@ -31,6 +31,16 @@ def test_warm_comparator_requires_authentication_for_writes assert_includes args, "--allow_unauthenticated_reads" end + def test_changed_source_registries_allow_reads_and_publication + args = ReapiRegistry.command(provider: "boringcache", phase: "commit", workspace: "boringcache/benchmarks", tag: "case-series") + refute_includes args, "--read-only" + assert_includes args, "--fail-on-cache-error" + args = ReapiRegistry.command(provider: "bazel-remote", phase: "commit", workspace: "boringcache/benchmarks", tag: "case-series") + refute_includes args, "--htpasswd_file" + assert_includes args, "reapi-store" + assert_raises(ReapiRegistry::Error) { ReapiRegistry.command(provider: "boringcache", phase: "unknown", workspace: "boringcache/benchmarks", tag: "case-series") } + end + def test_failed_shutdown_fails_after_a_successful_build Dir.mktmpdir do |directory| server = TCPServer.new("127.0.0.1", 0) diff --git a/test/reapi_report_test.rb b/test/reapi_report_test.rb index 4294ef96f..275eedeba 100644 --- a/test/reapi_report_test.rb +++ b/test/reapi_report_test.rb @@ -13,6 +13,8 @@ def test_phase_name_does_not_establish_a_cache_hit_or_miss assert_nil reported_cache_hit("warm", "0 remote cache hits") assert_equal true, reported_cache_hit("cold", "2 remote cache hits") assert_equal true, reported_cache_hit("warm", "81 remote cache hits") + assert_nil reported_cache_hit("commit", "0 remote cache hits") + assert_equal true, reported_cache_hit("commit", "2 remote cache hits") end private @@ -20,8 +22,8 @@ def test_phase_name_does_not_establish_a_cache_hit_or_miss def reported_cache_hit(phase, log) previous = ENV.to_h ENV.update("PHASE" => phase, "PROVIDER" => "bazel-remote", "BENCHMARK_ID" => "msgpack-sbt", - "CACHE_LANE" => "fresh", "BENCHMARK_SAMPLE" => "1", - "GITHUB_RUN_ID" => "1", "GITHUB_RUN_ATTEMPT" => "1") + "CACHE_LANE" => phase == "commit" ? "rolling" : "fresh", "BENCHMARK_SAMPLE" => "1", + "GITHUB_RUN_ID" => "1", "GITHUB_RUN_ATTEMPT" => "1", "GITHUB_REF_NAME" => "main") ENV.delete("BENCHMARK_SERIES_ID") Dir.mktmpdir do |directory| Dir.chdir(directory) do diff --git a/test/source_promotion_test.rb b/test/source_promotion_test.rb new file mode 100644 index 000000000..bc18da236 --- /dev/null +++ b/test/source_promotion_test.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true + +require "minitest/autorun" +require_relative "../scripts/source-promotion" + +class SourcePromotionTest < Minitest::Test + VERSION = "vcli-canary-0123456789ab" + + def proposal(item) + prefix = item.dig("execution", "source_prefix") + base = if prefix + BenchmarkPlan.settings(File.join(BenchmarkCases::ROOT, "cases", item.fetch("id"), "payload/benchmark-source.env")).fetch("#{prefix}_HEAD_SHA") + else + item.dig("source", "revision") || item.dig("source", "pins").find { |pin| pin["kind"] == "gitlink" }.fetch("revision") + end + record = {"case_id" => item.fetch("id"), "state" => "proposal", "base_sha" => base, "head_sha" => "a" * 40, + "source_distance" => "1", "requires_verified_build" => item.dig("execution", "sync") == "verified-pair"} + if prefix + text = File.read(File.join(BenchmarkCases::ROOT, "cases", item.fetch("id"), "payload/benchmark-source.env")) + record["candidate_source_env"] = text.sub(/^#{prefix}_BASE_SHA=.*$/, "#{prefix}_BASE_SHA=#{base}").sub(/^#{prefix}_HEAD_SHA=.*$/, "#{prefix}_HEAD_SHA=#{'a' * 40}") + end + record + end + + class Publisher < SourcePromotion::Publisher + attr_reader :saved, :requests + attr_accessor :fail_request, :outcome + + def initialize + super + @saved, @requests, @files = [], [], {} + @outcome = "in_progress" + end + + def optional_file(path, ref: "main") + @files[path] + end + + def save(record, expected:, changes: {}) + path = state_path(record.fetch("case_id")) + raise "State was overwritten" unless expected.fetch(path) == @files[path] + @saved << {"record" => Marshal.load(Marshal.dump(record)), "changes" => changes} + @files[path] = JSON.pretty_generate(record) + "\n" + end + + def api(path, body: nil) + if body + raise "Dispatch happened before a durable intent" unless @saved.last.dig("record", "state") == "dispatching" + @requests << body + return {} if @fail_request == @requests.length + {"workflow_run_id" => @requests.length} + else + {"status" => @outcome == "in_progress" ? "in_progress" : "completed", "conclusion" => @outcome} + end + end + end + + def test_snapshot_promotion_changes_only_source_pins_and_matching_nix_uri + item = BenchmarkCases.load_case("helix-nix") + candidate, changes = SourcePromotion.changes(item, proposal(item)) + assert_equal "a" * 40, candidate.dig("source", "revision") + assert_equal item.except("source"), candidate.except("source") + assert_equal %w[cases/helix-nix/case.json cases/helix-nix/payload/.boringcache.toml cases/helix-nix/payload/recipe-contract.json], changes.keys.sort + old = JSON.parse(File.read(File.join(BenchmarkCases::ROOT, "cases/helix-nix/payload/recipe-contract.json"))) + actual = JSON.parse(changes.fetch("cases/helix-nix/payload/recipe-contract.json")) + assert_equal old.fetch("upstream_files"), actual.fetch("upstream_files") + end + + def test_source_environment_cannot_change_toolchain_repository_or_commands + item = BenchmarkCases.load_case("deno") + proposed = proposal(item) + SourcePromotion.changes(item, proposed) + %w[DENO_RUST_VERSION DENO_SOURCE_REPOSITORY].each do |key| + changed = proposed.merge("candidate_source_env" => proposed.fetch("candidate_source_env").sub(/^#{key}=.*$/, "#{key}=changed")) + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, changed) } + end + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, proposed.merge("base_sha" => "b" * 40)) } + end + + def test_native_variants_and_zed_auto_are_the_only_selected_rolling_targets + item = BenchmarkCases.load_case("n8n") + targets = SourcePromotion.targets(item, proposal(item), version: VERSION) + assert_equal %w[distroless docker runners turbo], targets.map { |target| target.dig("inputs", "variant") }.sort + assert targets.all? { |target| target.dig("inputs", "cli_version") == VERSION } + zed = BenchmarkCases.load_case("zed") + targets = SourcePromotion.targets(zed, proposal(zed), version: VERSION) + assert_equal ["zed-zed-cargo-rolling-auto.yml"], targets.map { |target| target.fetch("workflow") } + end + + def test_every_scheduled_case_can_plan_a_changed_source_run + BenchmarkCadence.source_cases.each do |item| + targets = SourcePromotion.targets(item, proposal(item), version: VERSION) + refute_empty targets, item.fetch("id") + assert targets.all? { |target| target.fetch("lane") == "rolling" }, item.fetch("id") + end + end + + def test_proposal_cannot_disable_zed_build_verification + item = BenchmarkCases.load_case("zed") + assert_raises(SourcePromotion::Error) { SourcePromotion.changes(item, proposal(item).merge("requires_verified_build" => false)) } + end + + def test_request_failure_retains_other_variants_and_blocks_blind_retry + item = BenchmarkCases.load_case("n8n") + publisher = Publisher.new + publisher.fail_request = 2 + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + assert_equal 4, publisher.requests.length + record = JSON.parse(File.read(path)) + assert_equal "dispatch-failed", record.fetch("state") + assert_equal %w[requested request-unknown requested requested], record.fetch("runs").map { |run| run.fetch("state") } + assert publisher.saved.first.fetch("changes").key?("cases/n8n/case.json") + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + assert_equal 4, publisher.requests.length + end + end + + def test_zed_promotes_only_after_all_requested_builds_succeed + item = BenchmarkCases.load_case("zed") + publisher = Publisher.new + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + publisher.publish(item, proposal(item), version: VERSION, output: path) + assert_empty publisher.saved.first.fetch("changes") + assert_equal "requested", publisher.reconcile(item, output: path).fetch("state") + assert_equal 2, publisher.saved.length + publisher.outcome = "success" + assert_equal "promoted", publisher.reconcile(item, output: path).fetch("state") + assert publisher.saved.last.fetch("changes").key?("cases/zed/case.json") + assert publisher.saved.last.fetch("changes").key?("cases/zed/payload/benchmark-source.env") + end + end + + def test_failed_zed_build_preserves_the_declared_source + item = BenchmarkCases.load_case("zed") + publisher = Publisher.new + Dir.mktmpdir do |directory| + path = File.join(directory, "receipt.json") + publisher.publish(item, proposal(item), version: VERSION, output: path) + publisher.outcome = "failure" + assert_equal "failed", publisher.reconcile(item, output: path).fetch("state") + assert publisher.saved.all? { |entry| entry.fetch("changes").empty? } + assert_raises(SourcePromotion::Error) { publisher.publish(item, proposal(item), version: VERSION, output: path) } + end + end + + def test_publication_rejects_a_concurrent_edit_instead_of_overwriting_it + publisher = SourcePromotion::Publisher.new + requests = [] + api = lambda do |path, body: nil| + requests << path + path.include?("git/ref") ? {"object" => {"sha" => "b" * 40}} : {"type" => "file", "content" => Base64.strict_encode64("edited")} + end + publisher.stub(:api, api) do + assert_raises(SourcePromotion::Error) { publisher.commit({"cases/example/case.json" => "new"}, expected: {"cases/example/case.json" => "original"}, message: "Advance source") } + end + refute_includes requests, "graphql" + end + + def test_unrelated_branch_advancement_retries_with_a_new_expected_head + publisher = SourcePromotion::Publisher.new + heads = ["b" * 40, "c" * 40] + commits = [] + api = lambda do |path, body: nil| + if path == "graphql" + commits << body.fetch("variables").fetch("input") + commits.length == 1 ? {"errors" => [{"type" => "STALE_DATA", "message" => "Branch advanced"}]} : {"data" => {"createCommitOnBranch" => {"commit" => {"oid" => "d" * 40}}}} + elsif path.include?("git/ref") + {"object" => {"sha" => heads.shift}} + else + {"type" => "file", "content" => Base64.strict_encode64("original")} + end + end + publisher.stub(:api, api) do + assert_equal "d" * 40, publisher.commit({"cases/example/case.json" => "new"}, expected: {"cases/example/case.json" => "original"}, message: "Advance source") + end + assert_equal ["b" * 40, "c" * 40], commits.map { |input| input.fetch("expectedHeadOid") } + assert_equal "new", Base64.decode64(commits.last.dig("fileChanges", "additions", 0, "contents")) + end +end From a7d4c03f6e18931e07e20e6e9149c9ee30f99713 Mon Sep 17 00:00:00 2001 From: Gaurav Tiwari Date: Mon, 5 Oct 2026 15:02:32 +0100 Subject: [PATCH 22/22] Validate source controller revisions against trusted history --- .github/workflows/source-case.yml | 20 +++++++++- .../dispatches/1-nix-fresh-benchmark.yml.json | 37 +++++++++++++++++++ .../dispatches/1-nix-fresh-benchmark.yml.json | 37 +++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json create mode 100644 results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json diff --git a/.github/workflows/source-case.yml b/.github/workflows/source-case.yml index aa6181475..bf9cef333 100644 --- a/.github/workflows/source-case.yml +++ b/.github/workflows/source-case.yml @@ -58,8 +58,16 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: - ref: ${{ needs.previous.outputs.ref }} + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + fetch-depth: 0 persist-credentials: false + - name: Select the inspected revision from trusted history + env: + INSPECTED_REF: ${{ needs.previous.outputs.ref }} + run: | + [[ "$INSPECTED_REF" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git merge-base --is-ancestor "$INSPECTED_REF" HEAD + git checkout --detach "$INSPECTED_REF" - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - run: bundle install --jobs 4 --retry 2 - name: Inspect upstream against the reviewed recipe @@ -90,8 +98,16 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: - ref: ${{ needs.previous.outputs.ref }} + ref: ${{ vars.BENCHMARK_CADENCE_ACTIVE == 'true' && github.ref_name == 'main' && 'main' || github.sha }} + fetch-depth: 0 persist-credentials: false + - name: Select the inspected revision from trusted history + env: + INSPECTED_REF: ${{ needs.previous.outputs.ref }} + run: | + [[ "$INSPECTED_REF" =~ ^[0-9a-f]{40}$ ]] || exit 1 + git merge-base --is-ancestor "$INSPECTED_REF" HEAD + git checkout --detach "$INSPECTED_REF" - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b - run: bundle install --jobs 4 --retry 2 - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 diff --git a/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json b/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..220e76d8d --- /dev/null +++ b/results/helix-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "helix-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "helix-editor/helix", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "ba40e547426b0f9896c8bdc699a4ab11f2b37dbc" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "helix-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T13:57:41Z", + "ref": "shared-benchmark-cadence", + "state": "requested", + "run_id": 37320867908, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320867908" +} diff --git a/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json b/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json new file mode 100644 index 000000000..abfc0f3f0 --- /dev/null +++ b/results/zed-nix/cadence-fresh-01/dispatches/1-nix-fresh-benchmark.yml.json @@ -0,0 +1,37 @@ +{ + "case_id": "zed-nix", + "repository": "boringcache/benchmarks", + "workflow": "nix-fresh-benchmark.yml", + "workspace": "boringcache/benchmarks", + "source": { + "repository": "zed-industries/zed", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742", + "pins": [ + { + "path": "proposal.md", + "kind": "snapshot", + "revision": "96837d78cb0f2128c1965716f8df56b8aea59742" + } + ] + }, + "lane": "fresh", + "inputs": { + "case_id": "zed-nix", + "provider": "all", + "cli_version": "vcli-canary-7a5b27146ebe", + "cachix_cache": "benchmark-zed", + "series_id": "cadence-fresh-01", + "sample": "1" + }, + "phases": [ + "cold", + "warm" + ], + "variants": null, + "schema_version": 1, + "requested_at": "2026-10-05T13:57:42Z", + "ref": "shared-benchmark-cadence", + "state": "requested", + "run_id": 37320869828, + "run_url": "https://github.com/boringcache/benchmarks/actions/runs/37320869828" +}