From e48881edfe52fb81c7217f840595403baafaedf5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:53:04 +0000 Subject: [PATCH 1/3] Retry transient benchmark failures --- .github/workflows/benchmark-clawbench.yml | 5 ++-- benchmarks/harbor/README.md | 4 +++ benchmarks/harbor/clawbench/run.sh | 15 ++++++---- benchmarks/harbor/report.ts | 21 ++++++++++---- benchmarks/harbor/results.test.ts | 35 +++++++++++++++++++++-- benchmarks/harbor/results.ts | 2 ++ 6 files changed, 67 insertions(+), 15 deletions(-) diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 3d87ee99..ea703088 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -342,7 +342,8 @@ jobs: run_arm() { local checkout=$1 arm=$2 job_name=$3 set +e - "$checkout/benchmarks/harbor/clawbench/run.sh" \ + KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout" \ + "$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh" \ "$BENCHMARK_AGENT" \ "$BENCHMARK_TASK" \ "$job_name" \ @@ -479,4 +480,4 @@ jobs: [[ "$BASELINE_STATUS" == "0" ]] [[ "$PUBLISH_OUTCOME" == "success" ]] [[ "$REPORT_OUTCOME" == "success" ]] - jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null + jq -e 'all(.arms[]; .scored == .trials and .infraErrors == 0 and .ungraded == 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 771e7b6e..c4d23019 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -57,6 +57,8 @@ Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. +The runner retries a whole isolated trial up to five times for transient Hypeman connection, timeout, and exec-stream failures. Set `HARBOR_MAX_RETRIES` to override that limit. Per-request SDK retries remain disabled because transparently retrying instance or image creation can duplicate a request whose first response was lost. + ## GitHub Actions The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. Comparison runs benchmark the candidate SHA against its merge base so unrelated changes on the target branch do not affect the delta. Harbor, Hypeman, agent, and model versions are pinned by the workflow and each arm's observed agent configuration appears in the report. @@ -101,3 +103,5 @@ BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ ``` The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. + +Reports suppress comparison deltas when either arm has an infrastructure failure or ungraded trial. The workflow fails unless every intended trial is graded, while still retaining the incomplete report for diagnosis. diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 93a752a9..cdaa3806 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -15,10 +15,11 @@ agent=${1:-} [[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage task_id=${2:-v2-1134-chapter-finder-redcross} -repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) -benchmark_dir="$repo_root/benchmarks/harbor" -image_env="$benchmark_dir/.image.env" -clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +harness_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root} +benchmark_dir="$harness_root/benchmarks/harbor" +image_env="$source_root/benchmarks/harbor/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$harness_root/../ClawBench} clawbench_ref=${CLAWBENCH_REF:-c7feaa2435ca8115c0762c44e13885fe5adf3e98} [[ -f "$image_env" ]] || { @@ -161,6 +162,10 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_time --job-name "$job_name" \ --jobs-dir "$jobs_dir" \ --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \ - --max-retries 0 \ + --max-retries "${HARBOR_MAX_RETRIES:-5}" \ + --retry-include APITimeoutError \ + --retry-include APIConnectionError \ + --retry-include ConnectionRefusedError \ + --retry-include ExecProtocolError \ --delete \ --yes diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index cb91a6ce..fe22a091 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -89,12 +89,21 @@ export function renderMarkdown( ): string { const lines = ["", `## ${title}`]; const failed = Object.entries(statuses).filter(([, status]) => status !== 0); - const ungraded = summaries.filter((summary) => summary.scored === 0); - const incomplete = failed.length > 0 || ungraded.length > 0; + const infra = summaries.filter((summary) => summary.infraErrors > 0); + const ungraded = summaries.filter((summary) => summary.ungraded > 0); + const incomplete = + failed.length > 0 || infra.length > 0 || ungraded.length > 0; if (incomplete) { const reasons = [ ...failed.map(([arm, status]) => `${arm} exited ${status}`), - ...ungraded.map((summary) => `${summary.arm} produced no graded trials`), + ...infra.map( + (summary) => + `${summary.arm} had ${summary.infraErrors} infrastructure ${summary.infraErrors === 1 ? "failure" : "failures"}`, + ), + ...ungraded.map( + (summary) => + `${summary.arm} had ${summary.ungraded} ungraded ${summary.ungraded === 1 ? "trial" : "trials"}`, + ), ]; lines.push( "", @@ -103,12 +112,12 @@ export function renderMarkdown( } lines.push( "", - "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", - "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Retries | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", ); for (const summary of summaries) { lines.push( - `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, + `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.retries} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, ); } diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index 7634ad6d..656bb3a2 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -146,6 +146,7 @@ describe("Harbor result ingestion", () => { strict: 0, intercepted: 1, infraErrors: 1, + retries: 0, ungraded: 0, kernelMcpValid: 1, medianCalls: 1, @@ -303,8 +304,15 @@ describe("Harbor result ingestion", () => { { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials }, { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials }, ]); - expect(ungraded).toContain("candidate produced no graded trials"); + expect(ungraded).toContain("candidate had 2 ungraded trials"); expect(ungraded).not.toContain("Candidate minus baseline"); + + const infra = renderMarkdown("test", [ + { ...summary, arm: "candidate", infraErrors: 1 }, + { ...summary, arm: "baseline" }, + ]); + expect(infra).toContain("candidate had 1 infrastructure failure"); + expect(infra).not.toContain("Candidate minus baseline"); }); test("keeps full errors until redaction and clamps derived scores", () => { @@ -393,7 +401,9 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain( "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", ); - expect(workflow).toContain("all(.arms[]; .scored > 0)"); + expect(workflow).toContain( + "all(.arms[]; .scored == .trials and .infraErrors == 0 and .ungraded == 0)", + ); expect(workflow).toMatch( /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/, ); @@ -403,6 +413,27 @@ describe("benchmark workflow hardening", () => { expect(workflow).toContain( 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', ); + expect(workflow).toContain('KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout"'); + expect(workflow).toContain( + '"$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh"', + ); + + const runner = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"), + "utf8", + ); + expect(runner).toContain( + "source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root}", + ); + expect(runner).toContain('--max-retries "${HARBOR_MAX_RETRIES:-5}"'); + for (const exception of [ + "APITimeoutError", + "APIConnectionError", + "ConnectionRefusedError", + "ExecProtocolError", + ]) { + expect(runner).toContain(`--retry-include ${exception}`); + } }); test("requires the benchmark credential to resolve to one project", () => { diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts index dc24bab3..4c73a2eb 100644 --- a/benchmarks/harbor/results.ts +++ b/benchmarks/harbor/results.ts @@ -66,6 +66,7 @@ export interface ArmSummary { strict?: number; strictScored: number; infraErrors: number; + retries: number; ungraded: number; kernelMcpValid?: number; kernelMcpChecked: number; @@ -384,6 +385,7 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { strictScored: strict.length, infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") .length, + retries: arm.nRetries, ungraded: arm.trials.filter( (trial) => trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, From c413ad217e80193e001267c5a4e95b1f89de2b96 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:15:45 +0000 Subject: [PATCH 2/3] Use resilient Hypeman benchmark execution --- .github/workflows/benchmark-clawbench.yml | 2 +- benchmarks/harbor/README.md | 2 +- benchmarks/harbor/clawbench/run.sh | 4 +- benchmarks/harbor/publish-braintrust.ts | 14 ++-- benchmarks/harbor/report.ts | 13 +-- benchmarks/harbor/results.test.ts | 96 +++++++++++++++++------ benchmarks/harbor/results.ts | 32 ++++++-- 7 files changed, 115 insertions(+), 48 deletions(-) diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index ea703088..b0015bb5 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -480,4 +480,4 @@ jobs: [[ "$BASELINE_STATUS" == "0" ]] [[ "$PUBLISH_OUTCOME" == "success" ]] [[ "$REPORT_OUTCOME" == "success" ]] - jq -e 'all(.arms[]; .scored == .trials and .infraErrors == 0 and .ungraded == 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null + jq -e 'all(.arms[]; .complete == true)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index c4d23019..7b7b2ce2 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -16,7 +16,7 @@ The image records the current Git SHA, and the generated task records the ClawBe ## Requirements -- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 +- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.2 - Hypeman CLI credentials - a ClawBench checkout containing pinned commit `c7feaa2` - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index cdaa3806..1feda7e2 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -70,7 +70,7 @@ case "$agent" in esac harbor_version=${HARBOR_VERSION:-0.21.0} -harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1} +harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2} export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} @@ -165,6 +165,8 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_time --max-retries "${HARBOR_MAX_RETRIES:-5}" \ --retry-include APITimeoutError \ --retry-include APIConnectionError \ + --retry-include RateLimitError \ + --retry-include InternalServerError \ --retry-include ConnectionRefusedError \ --retry-include ExecProtocolError \ --delete \ diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index fc342d75..c6fa1c1b 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -354,13 +354,17 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { - const ungraded = arms + const incomplete = arms .map(summarizeArm) - .filter((summary) => summary.scored === 0) - .map((summary) => summary.arm); - if (ungraded.length > 0) { + .filter((summary) => !summary.complete); + if (incomplete.length > 0) { throw new Error( - `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`, + `Cannot publish incomplete benchmark arms: ${incomplete + .map( + (summary) => + `${summary.arm} (${summary.incompleteReasons.join(", ")})`, + ) + .join("; ")}`, ); } diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index fe22a091..f2d3c6cd 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -89,20 +89,13 @@ export function renderMarkdown( ): string { const lines = ["", `## ${title}`]; const failed = Object.entries(statuses).filter(([, status]) => status !== 0); - const infra = summaries.filter((summary) => summary.infraErrors > 0); - const ungraded = summaries.filter((summary) => summary.ungraded > 0); const incomplete = - failed.length > 0 || infra.length > 0 || ungraded.length > 0; + failed.length > 0 || summaries.some((summary) => !summary.complete); if (incomplete) { const reasons = [ ...failed.map(([arm, status]) => `${arm} exited ${status}`), - ...infra.map( - (summary) => - `${summary.arm} had ${summary.infraErrors} infrastructure ${summary.infraErrors === 1 ? "failure" : "failures"}`, - ), - ...ungraded.map( - (summary) => - `${summary.arm} had ${summary.ungraded} ungraded ${summary.ungraded === 1 ? "trial" : "trials"}`, + ...summaries.flatMap((summary) => + summary.incompleteReasons.map((reason) => `${summary.arm} ${reason}`), ), ]; lines.push( diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index 656bb3a2..b6fea01d 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -114,6 +114,24 @@ function fixture(): string { return root; } +function completeArm() { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const failed = arm.trials[1]; + failed.error = undefined; + failed.errorClass = undefined; + failed.rewards = { reward: 0, intercepted: 0 }; + failed.scores = { + accuracy: 0, + false_positive_rate: 0, + false_negative_rate: 1, + infra_error_rate: 0, + intercepted: 0, + reward: 0, + ungraded_rate: 0, + }; + return arm; +} + describe("Harbor result ingestion", () => { test("keeps infrastructure errors out of task-quality scores", () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); @@ -148,12 +166,33 @@ describe("Harbor result ingestion", () => { infraErrors: 1, retries: 0, ungraded: 0, + complete: false, + incompleteReasons: ["scored 1/2 trials", "had 1 infrastructure failure"], kernelMcpValid: 1, medianCalls: 1, totalCostUsd: 0.01, }); }); + test("requires every intended trial to be graded", () => { + expect(summarizeArm(completeArm()).complete).toBe(true); + + const missing = completeArm(); + missing.nTotalTrials = 3; + expect(summarizeArm(missing)).toMatchObject({ + complete: false, + incompleteReasons: ["scored 2/3 trials"], + }); + + const empty = completeArm(); + empty.nTotalTrials = 0; + empty.trials = []; + expect(summarizeArm(empty)).toMatchObject({ + complete: false, + incompleteReasons: ["had no intended trials"], + }); + }); + test("builds deterministic root, llm, and tool spans", () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); const first = buildExperimentEvents([arm], "test-experiment"); @@ -193,7 +232,7 @@ describe("Harbor result ingestion", () => { }); test("re-publishes the same rows and spans by deterministic ID", async () => { - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const arm = completeArm(); const originalFetch = globalThis.fetch; const inserts: string[][] = []; const metadataUpdates: unknown[] = []; @@ -259,14 +298,12 @@ describe("Harbor result ingestion", () => { } }); - test("does not publish arms without graded trials", async () => { + test("does not publish incomplete arms", async () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); for (const trial of arm.trials) trial.rewards = {}; await expect( publishBenchmark([arm], "project", "experiment", "api-key"), - ).rejects.toThrow( - "Cannot publish benchmark without graded trials for: candidate", - ); + ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate"); }); test("uses the lenient reward per trial and reports incomplete arms", () => { @@ -274,19 +311,7 @@ describe("Harbor result ingestion", () => { key: "reward_lenient", value: 1, }); - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); - const second = arm.trials[1]; - second.error = undefined; - second.errorClass = undefined; - second.rewards = { reward: 0 }; - second.scores = { - accuracy: 0, - false_positive_rate: 0, - false_negative_rate: 1, - infra_error_rate: 0, - reward: 0, - ungraded_rate: 0, - }; + const arm = completeArm(); const summary = summarizeArm(arm); expect(summary.scored).toBe(2); expect(summary.lenient).toBe(1); @@ -301,14 +326,34 @@ describe("Harbor result ingestion", () => { ]), ).toContain("+0.1 lenient"); const ungraded = renderMarkdown("test", [ - { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials }, - { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials }, + { + ...summary, + arm: "candidate", + scored: 0, + ungraded: summary.trials, + complete: false, + incompleteReasons: ["had 2 ungraded trials"], + }, + { + ...summary, + arm: "baseline", + scored: 0, + ungraded: summary.trials, + complete: false, + incompleteReasons: ["had 2 ungraded trials"], + }, ]); expect(ungraded).toContain("candidate had 2 ungraded trials"); expect(ungraded).not.toContain("Candidate minus baseline"); const infra = renderMarkdown("test", [ - { ...summary, arm: "candidate", infraErrors: 1 }, + { + ...summary, + arm: "candidate", + infraErrors: 1, + complete: false, + incompleteReasons: ["had 1 infrastructure failure"], + }, { ...summary, arm: "baseline" }, ]); expect(infra).toContain("candidate had 1 infrastructure failure"); @@ -401,9 +446,7 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain( "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", ); - expect(workflow).toContain( - "all(.arms[]; .scored == .trials and .infraErrors == 0 and .ungraded == 0)", - ); + expect(workflow).toContain("all(.arms[]; .complete == true)"); expect(workflow).toMatch( /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/, ); @@ -425,10 +468,15 @@ describe("benchmark workflow hardening", () => { expect(runner).toContain( "source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root}", ); + expect(runner).toContain( + "harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2}", + ); expect(runner).toContain('--max-retries "${HARBOR_MAX_RETRIES:-5}"'); for (const exception of [ "APITimeoutError", "APIConnectionError", + "RateLimitError", + "InternalServerError", "ConnectionRefusedError", "ExecProtocolError", ]) { diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts index 4c73a2eb..41def571 100644 --- a/benchmarks/harbor/results.ts +++ b/benchmarks/harbor/results.ts @@ -68,6 +68,8 @@ export interface ArmSummary { infraErrors: number; retries: number; ungraded: number; + complete: boolean; + incompleteReasons: string[]; kernelMcpValid?: number; kernelMcpChecked: number; medianCalls?: number; @@ -354,6 +356,26 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { }); const strict = numeric("reward_strict"); const validity = numeric("kernel_mcp_valid"); + const infraErrors = arm.trials.filter( + (trial) => trial.errorClass === "infra", + ).length; + const ungraded = arm.trials.filter( + (trial) => trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, + ).length; + const incompleteReasons = [ + ...(arm.nTotalTrials === 0 ? ["had no intended trials"] : []), + ...(primary.length !== arm.nTotalTrials + ? [`scored ${primary.length}/${arm.nTotalTrials} trials`] + : []), + ...(infraErrors > 0 + ? [ + `had ${infraErrors} infrastructure ${infraErrors === 1 ? "failure" : "failures"}`, + ] + : []), + ...(ungraded > 0 + ? [`had ${ungraded} ungraded ${ungraded === 1 ? "trial" : "trials"}`] + : []), + ]; const costs = arm.trials.flatMap((trial) => trial.metrics.costUsd === undefined ? [] : [trial.metrics.costUsd], ); @@ -383,13 +405,11 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { 0, ), strictScored: strict.length, - infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") - .length, + infraErrors, retries: arm.nRetries, - ungraded: arm.trials.filter( - (trial) => - trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, - ).length, + ungraded, + complete: incompleteReasons.length === 0, + incompleteReasons, kernelMcpValid: validity.length === 0 ? undefined From a56ca40933fc6d2ddcd00668cf79ff14671feb93 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:22:11 +0000 Subject: [PATCH 3/3] Pin resilient Hypeman environment in CI --- .github/workflows/benchmark-clawbench.yml | 2 +- benchmarks/harbor/results.test.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index b0015bb5..9f035ab4 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -220,7 +220,7 @@ jobs: BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} HARBOR_VERSION: "0.21.0" - HARBOR_HYPEMAN_VERSION: "0.1.1" + HARBOR_HYPEMAN_VERSION: "0.1.2" CODEX_BENCHMARK_MODEL: gpt-5.6-luna CODEX_BENCHMARK_VERSION: "0.120.0" CLAUDE_BENCHMARK_MODEL: claude-sonnet-5 diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index b6fea01d..a0774091 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -438,6 +438,7 @@ describe("benchmark workflow hardening", () => { expect(workflow).toContain("github.rest.repos.compareCommits"); expect(workflow).not.toContain("baseSha = pull.base.sha"); expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); + expect(workflow).toContain('HARBOR_HYPEMAN_VERSION: "0.1.2"'); expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); expect( workflow.match(/c7feaa2435ca8115c0762c44e13885fe5adf3e98/g),