From db83be8bb3be6f1f6d76b2534fbc54a5ceebc5c3 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 11:30:38 +0200 Subject: [PATCH 1/3] feat(eval): support reasoning-specific research plans --- scripts/native_eval/models.py | 12 ++++- scripts/native_eval/plan.py | 70 ++++++++++++++++++++++-- scripts/native_eval/run_job.py | 13 +++-- tests/test_native_eval_runner.py | 91 +++++++++++++++++++++++++++++++- 4 files changed, 176 insertions(+), 10 deletions(-) diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index 338b3e4..c0204ba 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -129,14 +129,22 @@ def build_matrix_plan( harnesses: Iterable[HarnessSpec] = HARNESSES, models: Iterable[ModelSpec] = MODELS, repetitions: Iterable[int] = (1, 2, 3), + reasoning_effort: str | None = None, ) -> list[RunSpec]: stamp = run_date or date.today().strftime("%Y%m%d") + repetition_values = tuple(repetitions) + if not repetition_values or any(value < 1 for value in repetition_values): + raise ValueError("repetitions must contain positive integers") + if len(set(repetition_values)) != len(repetition_values): + raise ValueError("repetitions must not contain duplicates") + reasoning_slug = reasoning_effort.replace("_", "-") if reasoning_effort else None plan: list[RunSpec] = [] for harness in harnesses: for model in models: - for repetition in repetitions: + for repetition in repetition_values: + reasoning_label = f"-{reasoning_slug}" if reasoning_slug else "" label = ( - f"{harness.name}-{model.slug}-full-" + f"{harness.name}-{model.slug}{reasoning_label}-full-" f"{expected_task_count}-r{repetition}-{stamp}" ) plan.append( diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index 06c7327..fca0bd9 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -2,12 +2,20 @@ import argparse from pathlib import Path +from typing import Sequence -from scripts.native_eval.models import build_matrix_plan +from scripts.native_eval.models import HARNESSES, MODELS, build_matrix_plan from scripts.native_eval.runtime import atomic_write_json, utc_now from scripts.native_eval.tasks import validate_suite +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + def write_run_index( *, tasks_root: Path, @@ -16,13 +24,39 @@ def write_run_index( run_date: str, reasoning_effort: str, judge_reasoning_effort: str, + judge_model_id: str = "gpt-5.6-sol", + repetitions: int = 3, + harness_names: Sequence[str] | None = None, + model_slugs: Sequence[str] | None = None, ) -> list[dict[str, object]]: tasks = validate_suite(tasks_root) - runs = build_matrix_plan(len(tasks), run_date=run_date) + selected_harnesses = tuple( + harness + for harness in HARNESSES + if harness_names is None or harness.name in harness_names + ) + selected_models = tuple( + model + for model in MODELS + if model_slugs is None or model.slug in model_slugs + ) + if not selected_harnesses: + raise ValueError("no harnesses selected") + if not selected_models: + raise ValueError("no models selected") + runs = build_matrix_plan( + len(tasks), + run_date=run_date, + harnesses=selected_harnesses, + models=selected_models, + repetitions=range(1, repetitions + 1), + reasoning_effort=reasoning_effort, + ) entries = [ { **run.to_dict(), "reasoning_effort": reasoning_effort, + "judge_model_id": judge_model_id, "judge_reasoning_effort": judge_reasoning_effort, "attempt": 0, "status": "planned", @@ -41,13 +75,19 @@ def write_run_index( "task_suite_path": "combined tasks/tasks", "expected_task_count": len(tasks), "planned_run_count": len(entries), + "repetition_count": repetitions, + "reasoning_effort": reasoning_effort, + "judge_model_id": judge_model_id, + "judge_reasoning_effort": judge_reasoning_effort, + "harnesses": [harness.name for harness in selected_harnesses], + "models": [model.slug for model in selected_models], "runs": entries, }, ) return entries -def main() -> None: +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--tasks-root", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) @@ -62,16 +102,38 @@ def main() -> None: "--judge-reasoning-effort", choices=("low", "medium", "high", "xhigh"), ) - args = parser.parse_args() + parser.add_argument("--judge-model-id", default="gpt-5.6-sol") + parser.add_argument("--repetitions", type=_positive_int, default=3) + parser.add_argument( + "--harness", + action="append", + choices=tuple(harness.name for harness in HARNESSES), + dest="harness_names", + ) + parser.add_argument( + "--model", + action="append", + choices=tuple(model.slug for model in MODELS), + dest="model_slugs", + ) + return parser.parse_args(argv) + + +def main() -> None: + args = parse_args() entries = write_run_index( tasks_root=args.tasks_root, output=args.output, public_tasks_commit=args.public_tasks_commit, run_date=args.run_date, reasoning_effort=args.reasoning_effort, + judge_model_id=args.judge_model_id, judge_reasoning_effort=( args.judge_reasoning_effort or args.reasoning_effort ), + repetitions=args.repetitions, + harness_names=args.harness_names, + model_slugs=args.model_slugs, ) print(f"wrote {len(entries)} planned runs to {args.output}") diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 5658e8d..d12935f 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -350,7 +350,14 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: ) -def parse_args() -> argparse.Namespace: +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--tasks-root", type=Path, required=True) parser.add_argument("--jobs-dir", type=Path, required=True) @@ -361,7 +368,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--model-id") parser.add_argument("--model-provider") parser.add_argument("--proxy-model-name") - parser.add_argument("--repetition", type=int, choices=(1, 2, 3), required=True) + parser.add_argument("--repetition", type=_positive_int, required=True) parser.add_argument("--expected-task-count", type=int, required=True) parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--task-suite-path", required=True) @@ -383,7 +390,7 @@ def parse_args() -> argparse.Namespace: help="Run only the named task. Repeat for multiple tasks.", ) parser.add_argument("--rerun-of-canonical-run") - return parser.parse_args() + return parser.parse_args(argv) def main() -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 34f13f7..10cea4d 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -25,7 +25,12 @@ ) from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config -from scripts.native_eval.run_job import _git_commit, _run_manifest, build_run_spec +from scripts.native_eval.run_job import ( + _git_commit, + _run_manifest, + build_run_spec, + parse_args as parse_run_job_args, +) from scripts.native_eval.runtime import ( DockerTaskEnvironment, build_judge_env, @@ -64,7 +69,91 @@ def test_run_index_records_agent_and_judge_reasoning( assert len(entries) == 96 assert {entry["reasoning_effort"] for entry in entries} == {"high"} + assert {entry["judge_model_id"] for entry in entries} == {"gpt-5.6-sol"} assert {entry["judge_reasoning_effort"] for entry in entries} == {"high"} + assert all("-high-full-" in str(entry["run_label"]) for entry in entries) + + +def test_run_index_supports_six_repetitions_and_filters( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setattr(native_plan, "validate_suite", lambda _root: [object()]) + + entries = native_plan.write_run_index( + tasks_root=tmp_path, + output=tmp_path / "run-index.json", + public_tasks_commit="tasks-commit", + run_date="20260729", + reasoning_effort="medium", + judge_reasoning_effort="high", + repetitions=6, + harness_names=["openclaw", "hermes"], + model_slugs=["gpt56-sol"], + ) + + assert len(entries) == 12 + assert {entry["harness"] for entry in entries} == {"openclaw", "hermes"} + assert {entry["model_slug"] for entry in entries} == {"gpt56-sol"} + assert {entry["repetition"] for entry in entries} == set(range(1, 7)) + assert len({entry["run_label"] for entry in entries}) == 12 + assert all("-medium-full-" in str(entry["run_label"]) for entry in entries) + + +def test_plan_cli_accepts_repetition_count_and_filters() -> None: + args = native_plan.parse_args( + [ + "--tasks-root", + "tasks", + "--output", + "run-index.json", + "--public-tasks-commit", + "abc", + "--run-date", + "20260729", + "--reasoning-effort", + "low", + "--repetitions", + "6", + "--harness", + "openclaw", + "--model", + "gpt56-sol", + ] + ) + + assert args.repetitions == 6 + assert args.harness_names == ["openclaw"] + assert args.model_slugs == ["gpt56-sol"] + + +def test_run_job_cli_accepts_repetition_six() -> None: + args = parse_run_job_args( + [ + "--tasks-root", + "tasks", + "--jobs-dir", + "jobs", + "--run-label", + "openclaw-gpt56-sol-high-full-115-r6-20260729", + "--harness", + "openclaw", + "--model-slug", + "gpt56-sol", + "--repetition", + "6", + "--expected-task-count", + "115", + "--public-tasks-commit", + "abc", + "--task-suite-path", + "combined tasks/tasks", + "--run-date", + "20260729", + ] + ) + + assert args.repetition == 6 def test_task_loader_accepts_rich_manifest_and_compose(tmp_path: Path) -> None: From ae3cb37c6de56e912bb879172ec2bd6bd19ffce6 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 11:30:40 +0200 Subject: [PATCH 2/3] feat(eval): add trace-audited research runbook --- .../shellbench-research-runbook/SKILL.md | 78 +++ .../agents/openai.yaml | 7 + .../references/runbook.md | 462 +++++++++++++ CHANGELOG.md | 3 + scripts/native_eval/research_audit.py | 632 ++++++++++++++++++ tests/test_native_eval_research_audit.py | 180 +++++ 6 files changed, 1362 insertions(+) create mode 100644 .agents/skills/shellbench-research-runbook/SKILL.md create mode 100644 .agents/skills/shellbench-research-runbook/agents/openai.yaml create mode 100644 .agents/skills/shellbench-research-runbook/references/runbook.md create mode 100644 scripts/native_eval/research_audit.py create mode 100644 tests/test_native_eval_research_audit.py diff --git a/.agents/skills/shellbench-research-runbook/SKILL.md b/.agents/skills/shellbench-research-runbook/SKILL.md new file mode 100644 index 0000000..3f89273 --- /dev/null +++ b/.agents/skills/shellbench-research-runbook/SKILL.md @@ -0,0 +1,78 @@ +--- +name: shellbench-research-runbook +description: Plan, smoke-test, execute, checkpoint, publish, audit, and reproduce full ShellBench native benchmark campaigns across OpenClaw, Hermes, Codex, and Claude Code, including model and reasoning identity, pinned harness versions, n=3 qualification through n=6 research runs, S3 trace retention, and task-turn-tool-token-cost exports. +--- + +# ShellBench Research Runbook + +Use this skill for a real benchmark campaign, not a one-off local score. + +Read [references/runbook.md](references/runbook.md) before provisioning machines. +It is the normative campaign contract and contains the commands, gates, artifact +schema, and recovery rules. + +## Non-negotiable gates + +1. Use remote Crabbox AWS beasts for benchmark execution. Never run scored + trials on the operator laptop. +2. Pin one public-task commit, runner commit or patch hash, provider model ID, + harness version, reasoning level, and judge route for the whole campaign. +3. Run three-task smokes for every distinct harness, model, and reasoning route. + Do not start full-suite jobs until model identity, real traces, tools, usage, + judge routing, and artifact export pass. +4. Qualify with independent repetitions `r1` through `r3`. After a clean audit, + add `r4` through `r6`; the research result is six total repetitions. +5. Run every provider-supported non-maximum reasoning level. Never label a + reasoning level as tested unless the route applies it and the trace or proxy + evidence proves it. Record unsupported levels instead of fabricating them. +6. Use `gpt-5.6-sol` at `high` as the default judge. Keep the judge alias, + credentials, logs, and identity audit separate from the agent route. +7. Start checkpointing after the first completed trial and continue at least + every ten minutes or ten new results. Verify each local archive before it + counts. +8. Upload every verified checkpoint and final archive to the private S3 prefix + from `SHELLBENCH_TRACE_S3_URI`. Never put bucket names or credentials in git, + PR text, public logs, or generated reports. +9. A run is not research-clean when traces are missing, observed model identity + differs from the request, reasoning is unproven, coverage is incomplete, or + infrastructure failures dominate. + +## Required commands + +Generate reasoning-specific plans with unique labels: + +```sh +python -m scripts.native_eval.plan \ + --tasks-root "$TASKS_ROOT" \ + --output "$CAMPAIGN/manifests/run-index-high.json" \ + --public-tasks-commit "$PUBLIC_TASKS_COMMIT" \ + --run-date "$RUN_DATE" \ + --reasoning-effort high \ + --judge-model-id gpt-5.6-sol \ + --judge-reasoning-effort high \ + --repetitions 3 +``` + +Use repeatable `--harness` and `--model` filters for smoke or phased plans. +Set `--repetitions 6` only after the first three repetitions pass qualification. + +After extraction, produce the research tables and strict identity report: + +```sh +python -m scripts.native_eval.research_audit \ + --run-index "$CAMPAIGN/manifests/run-index.json" \ + --extracted-root "$CAMPAIGN/extracted" \ + --output-dir "$CAMPAIGN/summaries/research" +``` + +Treat any false row in `model_identity_audit.csv` as a blocker. The exporter +does not invent prices: missing exact spend remains explicitly unavailable. + +## Stop conditions + +- Stop a route after any smoke contains the wrong or multiple model IDs. +- Stop when the judge's observed provider model cannot be proven. +- Stop when a harness version differs from its campaign pin. +- Preserve and exclude infra-dominated runs, then rerun the same repetition + with a suffix and lower concurrency. +- Never delete, overwrite, or silently replace an artifact or repetition. diff --git a/.agents/skills/shellbench-research-runbook/agents/openai.yaml b/.agents/skills/shellbench-research-runbook/agents/openai.yaml new file mode 100644 index 0000000..609e440 --- /dev/null +++ b/.agents/skills/shellbench-research-runbook/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "ShellBench Research Runbook" + short_description: "Run reproducible, trace-audited benchmark campaigns" + default_prompt: "Use $shellbench-research-runbook to plan and audit a full native ShellBench research campaign." + +policy: + allow_implicit_invocation: true diff --git a/.agents/skills/shellbench-research-runbook/references/runbook.md b/.agents/skills/shellbench-research-runbook/references/runbook.md new file mode 100644 index 0000000..4526d67 --- /dev/null +++ b/.agents/skills/shellbench-research-runbook/references/runbook.md @@ -0,0 +1,462 @@ +# Full Native ShellBench Research Campaign + +## 2026-07-29 Revision Appendix + +This replaces the original full-matrix prompt with stronger research gates: +three-task route smokes, verified private S3 retention, strict observed-model +audits, requested-versus-installed harness versions, n=3 qualification followed +by n=6 total repetitions, every provider-supported non-maximum reasoning level, +`gpt-5.6-sol` at `high` as judge, and task/turn/tool/token/cost exports with +explicit evidence provenance. + +## Objective + +Run the current ShellBench combined public-task suite natively on remote +Crabbox AWS beasts across the selected harnesses, provider model IDs, and +reasoning levels. Preserve enough evidence to reproduce every score and audit +which model, harness build, tools, tokens, costs, judge, task commit, and runner +code produced it. + +The campaign progresses through three gates: + +1. route smoke: three representative tasks, one repetition +2. qualification: full suite, independent repetitions `r1` through `r3` +3. research completion: add `r4` through `r6` after qualification is clean + +`n=6` means six independent jobs and writable environments. It does not mean +rerunning one job directory or replaying saved agent state. + +## Scope + +Harnesses: + +- `openclaw` +- `hermes` +- `codex` +- `claude-code` + +The model list is campaign input. Resolve every friendly name to an actual +provider model ID from the live catalog before planning. Never infer or silently +substitute an ID. + +Reasoning policy: + +- enumerate every level the provider and route actually support +- run all supported non-maximum levels +- exclude provider levels named or documented as `max` or `ultra` +- the default OpenAI research set is `low`, `medium`, and `high` +- do not generate low/medium/high variants for a provider whose adapter ignores + the setting +- record `unsupported` or `unverified` in the campaign manifest instead + +Default judge: + +```text +provider model ID: gpt-5.6-sol +reasoning: high +``` + +The judge must use a dedicated proxy alias. Agent and judge requests must be +distinguishable in proxy/provider logs. + +## Current Toolchain Pins + +The code source of truth is `scripts/native_eval/models.py` and +`scripts/native_eval/bootstrap_beast.sh`. At this runbook revision the pins are: + +| Component | Pin | +|---|---| +| OpenClaw | `2026.7.1-2` | +| Hermes | `cb06017b1d6e1b9ae0cb35f99a48ffa6bcbaa828` | +| Codex | `0.145.0` | +| Claude Code | `2.1.220` | +| Node | `22.23.1` | +| LiteLLM | `1.93.0` | + +Before every campaign: + +1. compare this table with the code pins +2. record requested pins in the campaign manifest +3. record installed versions from `/opt/shellbench-native/manifest.json` +4. fail smoke when requested and installed versions differ + +Do not update a pin during a campaign. Start a new campaign ID when a harness, +runner, task, provider route, or judge version changes. + +## Campaign Identity + +Use a stable campaign ID: + +```text +shellbench-full--- +``` + +Reasoning-specific run labels are mandatory: + +```text +---full--r- +``` + +Examples: + +```text +openclaw-gpt56-sol-low-full-115-r1-20260729 +hermes-gpt56-sol-high-full-115-r6-20260729 +``` + +Smokes use: + +```text +---smoke--r1- +``` + +Retries append a suffix and never replace the original: + +```text +-rerun1 +-c8 +-infra-timeout +``` + +## Preflight + +Run locally without printing secrets: + +```sh +crabbox whoami +crabbox --version +test -f .env +``` + +If auth is expired: + +```sh +crabbox login +``` + +Check only whether required environment variables are present. Do not echo +their values. + +Fetch and pin the task suite: + +```sh +git -C work/public-tasks fetch origin +PUBLIC_TASKS_COMMIT="$(git -C work/public-tasks rev-parse origin/main)" +TASK_SUITE_PATH="combined tasks/tasks" +EXPECTED_TASK_COUNT="$( + git -C work/public-tasks ls-tree -d --name-only \ + "$PUBLIC_TASKS_COMMIT:$TASK_SUITE_PATH" | wc -l | tr -d ' ' +)" +test "$EXPECTED_TASK_COUNT" -gt 0 +``` + +Validate every immediate task directory with +`scripts.native_eval.tasks.validate_suite`. The count must be derived from the +pinned commit, never copied from an older campaign. + +Create and verify the immutable suite archive: + +```sh +git -C work/public-tasks archive "$PUBLIC_TASKS_COMMIT" -- "$TASK_SUITE_PATH" \ + > public-tasks-main-combined.tar +gzip -f public-tasks-main-combined.tar + +ARCHIVE_TASK_COUNT="$( + tar -tzf public-tasks-main-combined.tar.gz | + awk -F/ '$1=="combined tasks" && $2=="tasks" && $3!="" {print $3}' | + sort -u | wc -l | tr -d ' ' +)" +test "$ARCHIVE_TASK_COUNT" = "$EXPECTED_TASK_COUNT" +``` + +Record: + +- public-tasks commit and suite path +- expected task count and archive SHA-256 +- ShellBench runner commit and dirty patch hash +- every harness requested pin and installed version +- Crabbox CLI version +- lease ID, slug, instance type, IP, region, and timestamps +- friendly model slug, provider, requested provider ID, and proxy alias +- requested reasoning and whether the adapter can prove it +- judge provider ID, reasoning, proxy alias, and observed identity evidence + +## Phase 1: Route Smokes + +Select three tasks from the pinned suite: + +1. a simple shell or filesystem task +2. a browser, app, or multi-container task +3. a judge-backed or semantically verified task + +Record task names and checksums. Do not hard-code task names in the runbook +because the public suite changes. + +Run each distinct harness, model, and reasoning route with: + +- one fresh beast or isolated job +- repetition `1` +- concurrency `1` or `2` +- the same proxy and judge configuration intended for the full campaign +- checkpointing enabled immediately + +Smokes may run in parallel across beasts. Do not increase per-smoke concurrency; +the purpose is routing and evidence validation, not throughput. + +For every smoke, verify: + +- all three task results exist +- `trajectory_status` is `real` for supported harnesses +- raw harness events and normalized `trajectory.json` are both retained +- observed agent model IDs equal exactly the requested provider model ID +- no hidden fallback, mixed model, or alias substitution appears +- requested reasoning is present in proxy request evidence +- installed harness version equals the pin +- tool calls and observations are represented in the trace +- task-level token totals exist or are explicitly unavailable +- exact provider cost exists or is explicitly unavailable +- judge requests use only `gpt-5.6-sol` at `high` +- checkpoint and final archives pass local `tar -tzf` +- verified archives upload to S3 and can be read back or headed + +The agent trace alone cannot prove judge identity. Preserve proxy/provider +request logs and audit the dedicated judge alias during smoke. + +Any wrong, mixed, or unobserved model ID blocks that route. Fix the route and +repeat the smoke under a new label. + +## Phase 2: Qualification At n=3 + +Generate one plan per reasoning level so labels remain unique: + +```sh +python -m scripts.native_eval.plan \ + --tasks-root "$TASKS_ROOT" \ + --output "$CAMPAIGN/manifests/run-index-$REASONING.json" \ + --public-tasks-commit "$PUBLIC_TASKS_COMMIT" \ + --run-date "$RUN_DATE" \ + --reasoning-effort "$REASONING" \ + --judge-model-id gpt-5.6-sol \ + --judge-reasoning-effort high \ + --repetitions 3 +``` + +Use repeatable filters to phase the fleet: + +```sh +--harness openclaw --harness hermes +--model gpt56-sol +``` + +Each repetition must have: + +- a unique job directory +- a fresh writable task environment +- no reused agent session +- no copied response cache +- independent checkpoints and final archive + +Prefer one AWS beast per run when quota allows. Otherwise run waves. Start +browser and app-heavy routes at task concurrency `16`; lower to `8` after +startup pressure, or raise a proven stable route to `32`. Do not use `96+` +except as a named infrastructure stress experiment. + +## Phase 3: Research Completion At n=6 + +Only after all three qualification repetitions are complete and clean: + +1. regenerate or extend the plan to repetitions `r1` through `r6` +2. preserve the completed `r1` through `r3` entries and artifacts +3. schedule only new `r4` through `r6` jobs +4. audit all six together + +Do not treat three clean runs plus three replacement runs as six independent +clean repetitions unless all six original run identities and artifacts remain. + +## Checkpoint And Final Retention + +For every active run: + +1. pull after the first completed trial +2. pull at least every ten minutes or ten new completed trials +3. pull immediately on stalled progress, repeated infra errors, degraded lease + health, or operator disconnect +4. make a final pull after the process exits, including failures + +Names are monotonic: + +```text +-checkpoint-0001-artifacts.tar.gz +-checkpoint-0002-artifacts.tar.gz +-final-artifacts.tar.gz +``` + +After every local copy: + +```sh +tar -tzf "$LOCAL_ARCHIVE" >/dev/null +RESULT_COUNT="$( + tar -tzf "$LOCAL_ARCHIVE" | + awk '/\/result\.json$/ {count++} END {print count+0}' +)" +shasum -a 256 "$LOCAL_ARCHIVE" +``` + +Keep all verified checkpoints even after a successful final export. + +The archive must contain: + +- run and trial `result.json` +- `run_manifest.json`, `config.json`, and lock/state metadata +- raw harness sessions and event streams +- normalized trajectories +- stdout, stderr, setup, proxy, and verifier logs +- provider/proxy usage and spend records +- task checksums and suite commit +- harness/toolchain installed-version manifest + +## Private S3 Publication + +The destination comes only from: + +```text +SHELLBENCH_TRACE_S3_URI +``` + +It is a private `s3://` prefix supplied outside git. Do not place its value in +the run index, generated Markdown, PR text, shell history, or logs. + +Upload only locally verified archives. Store: + +```text +//raw/ +//manifests/.sha256 +//manifests/upload-index.json +``` + +When `clawbench traces upload` is available, prefer it because it records +SHA-256 metadata and verifies the remote object. Otherwise use AWS CLI with +server-side encryption, write a checksum sidecar, and verify the remote object +metadata or read-back before marking the upload complete. + +Never delete the local verified copy merely because S3 upload succeeded. + +## Model And Reasoning Audit + +After extracting artifacts: + +```sh +python -m scripts.native_eval.research_audit \ + --run-index "$CAMPAIGN/manifests/run-index.json" \ + --extracted-root "$CAMPAIGN/extracted" \ + --output-dir "$CAMPAIGN/summaries/research" +``` + +The command writes: + +- `trace_inventory.csv`: one row per task result +- `model_identity_audit.csv`: one strict identity summary per run +- `turn_usage.csv`: one row per normalized trace step +- `tool_calls.csv`: one row per tool call +- `research_audit.json`: counts and audit status + +Identity passes only when every recovered task trace observes exactly the +requested model ID. Missing traces and missing observed identity fail the +audit; they are not treated as neutral. + +Also audit proxy/provider logs: + +- agent alias resolves only to the requested provider model ID +- judge alias resolves only to `gpt-5.6-sol` +- requested reasoning is present on each applicable request +- no fallback or retry changes the provider model ID +- request IDs can be joined to run, task, and turn where available + +## Tokens, Tools, And Cost + +Task-level token totals can be recovered when the harness emits them in +`agent_result` or ATIF `final_metrics`. Tool calls can be recovered per turn +from `steps[].tool_calls`. + +Per-turn token and cost analysis has three evidence levels: + +| Level | Requirement | Report as | +|---|---|---| +| Exact | per-request usage/spend in trace or proxy/provider log | `exact_*` | +| Estimated | tokens plus pinned provider ID and dated pricing snapshot | `estimated` | +| Missing | neither exact spend nor sufficient pricing evidence | `unavailable_*` | + +Never call a pricing reconstruction exact. Pin the pricing snapshot date, +currency, input/cache/output/reasoning rates, and source beside any estimate. + +For exact research accounting, preserve a request-level spend record containing: + +- request ID +- run label and task name +- harness and provider model ID +- requested reasoning +- input, cached input, output, and reasoning tokens +- provider-reported cost +- started and finished timestamps +- agent versus judge role + +Per-tool monetary cost is not an LLM trace field. Report tool count, duration, +and failures from traces; calculate tool infrastructure cost only from separate +runtime, API billing, or machine accounting evidence. + +## Failure Policy + +Classify: + +- `infra`: environment, Docker, setup, gateway, or provider connectivity +- `agent_exit`: nonzero agent exit not caused by infrastructure +- `verifier_missing_reward`: missing reward file +- `clean_fail`: zero reward without exception +- `partial`: reward between zero and one +- `pass`: reward at least one + +Preserve every failure. Infra-dominated or identity-invalid runs are excluded, +not discarded. Rerun the same repetition under a suffixed label after fixing +the cause. + +## Deliverables + +Local campaign layout: + +```text +runs-full-YYYYMMDD/ + raw/ + extracted/ + logs/ + manifests/ + campaign_manifest.json + run_index.json + upload-index.json + summaries/ + aggregate_results.csv + aggregate_results.json + per_task_results.csv + infra_failures.csv + cleaned_leaderboard.md + research/ + trace_inventory.csv + model_identity_audit.csv + turn_usage.csv + tool_calls.csv + research_audit.json +``` + +The audit note must state: + +- clean, excluded, missing, and rerun-required repetitions +- full task coverage against the pinned task commit +- requested and observed model IDs +- requested and proven reasoning levels +- requested and installed harness versions +- judge identity evidence +- exact, estimated, and unavailable usage/cost coverage +- local and S3 artifact counts plus checksum verification status + +Do not call the comparison fair unless coverage is full, identity is proven, +the same task commit and judge contract were used, and infrastructure failures +are low. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0815d4f..cb33aeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,3 +29,6 @@ - Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog. - Report repair-overlay task provenance and original-versus-repaired score sensitivity when the original job is available. +- Add a research campaign skill with smoke gates, n=3 through n=6 execution, + private trace retention, strict model-identity checks, pinned harness + versions, and task/turn/tool/token/cost analysis exports. diff --git a/scripts/native_eval/research_audit.py b/scripts/native_eval/research_audit.py new file mode 100644 index 0000000..2ab8e34 --- /dev/null +++ b/scripts/native_eval/research_audit.py @@ -0,0 +1,632 @@ +"""Export task, turn, tool, usage, and model-identity research tables.""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +from collections import Counter +from pathlib import Path +from typing import Any, Iterable + + +TRACE_FIELDS = ( + "run_label", + "harness", + "harness_version", + "installed_harness_version", + "harness_version_status", + "model_slug", + "expected_model_id", + "reasoning_effort", + "judge_model_id", + "judge_reasoning_effort", + "judge_identity_status", + "repetition", + "task_name", + "reward", + "result_path", + "trajectory_path", + "toolchain_manifest_path", + "proxy_log_path", + "trajectory_status", + "observed_model_ids", + "model_identity_status", + "turn_count", + "tool_call_count", + "n_input_tokens", + "n_cache_tokens", + "n_output_tokens", + "cost_usd", + "cost_provenance", +) + +TURN_FIELDS = ( + "run_label", + "harness", + "model_slug", + "expected_model_id", + "reasoning_effort", + "repetition", + "task_name", + "turn_index", + "source", + "timestamp", + "message_chars", + "reasoning_chars", + "tool_call_count", + "n_input_tokens", + "n_cache_tokens", + "n_output_tokens", + "cost_usd", + "cost_provenance", + "trajectory_path", +) + +TOOL_FIELDS = ( + "run_label", + "harness", + "model_slug", + "expected_model_id", + "reasoning_effort", + "repetition", + "task_name", + "turn_index", + "tool_index", + "tool_call_id", + "function_name", + "arguments_json", + "observation_chars", + "observation_excerpt", + "trajectory_path", +) + +RUN_AUDIT_FIELDS = ( + "run_label", + "harness", + "harness_version", + "installed_harness_version", + "harness_version_status", + "model_slug", + "expected_model_id", + "reasoning_effort", + "judge_model_id", + "judge_reasoning_effort", + "judge_identity_status", + "repetition", + "expected_task_count", + "result_count", + "real_trace_count", + "identity_match_count", + "identity_mismatch_count", + "identity_not_observed_count", + "trace_missing_count", + "model_identity_audit_passed", + "toolchain_manifest_path", + "proxy_log_path", + "task_cost_exact_count", + "task_cost_unavailable_count", +) + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _number(value: Any) -> int | float | None: + if value is None or isinstance(value, bool): + return None + try: + parsed = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(parsed): + return None + return int(parsed) if parsed.is_integer() else parsed + + +def _nested(mapping: dict[str, Any], *keys: str) -> Any: + value: Any = mapping + for key in keys: + if not isinstance(value, dict): + return None + value = value.get(key) + return value + + +def _first_number(mapping: dict[str, Any], keys: Iterable[str]) -> int | float | None: + for key in keys: + value = _number(mapping.get(key)) + if value is not None: + return value + return None + + +def _usage( + agent_result: dict[str, Any], + trajectory: dict[str, Any], +) -> tuple[int | float | None, int | float | None, int | float | None]: + metrics = trajectory.get("final_metrics") + if not isinstance(metrics, dict): + metrics = {} + return ( + _first_number( + agent_result, + ("n_input_tokens", "input_tokens", "prompt_tokens"), + ) + or _first_number(metrics, ("total_prompt_tokens", "prompt_tokens")), + _first_number( + agent_result, + ("n_cache_tokens", "cache_tokens", "cached_tokens"), + ) + or _first_number(metrics, ("total_cached_tokens", "cached_tokens")), + _first_number( + agent_result, + ("n_output_tokens", "output_tokens", "completion_tokens"), + ) + or _first_number(metrics, ("total_completion_tokens", "completion_tokens")), + ) + + +def _cost( + agent_result: dict[str, Any], + trajectory: dict[str, Any], +) -> tuple[int | float | None, str]: + cost = _first_number( + agent_result, + ("cost_usd", "total_cost_usd", "total_cost"), + ) + if cost is not None: + return cost, "exact_harness" + metrics = trajectory.get("final_metrics") + if isinstance(metrics, dict): + cost = _first_number(metrics, ("total_cost_usd", "cost_usd", "total_cost")) + if cost is not None: + return cost, "exact_trace" + return None, "unavailable_without_provider_spend_or_pricing_snapshot" + + +def _step_usage(step: dict[str, Any]) -> tuple[Any, Any, Any, Any, str]: + metrics: dict[str, Any] = {} + for key in ("usage", "token_usage", "metrics"): + candidate = step.get(key) + if isinstance(candidate, dict): + metrics.update(candidate) + input_tokens = _first_number( + metrics, + ("input_tokens", "prompt_tokens", "total_prompt_tokens"), + ) + cache_tokens = _first_number( + metrics, + ("cache_read_input_tokens", "cached_tokens", "total_cached_tokens"), + ) + output_tokens = _first_number( + metrics, + ("output_tokens", "completion_tokens", "total_completion_tokens"), + ) + cost = _first_number(metrics, ("cost_usd", "total_cost_usd", "total_cost")) + provenance = ( + "exact_trace_turn" + if cost is not None + else "unavailable_without_per_request_spend" + ) + return input_tokens, cache_tokens, output_tokens, cost, provenance + + +def _task_name(result: dict[str, Any], result_path: Path) -> str: + task_id = result.get("task_id") + if isinstance(task_id, dict): + path = task_id.get("path") + if isinstance(path, str) and path: + return Path(path).name + name = task_id.get("name") + if isinstance(name, str) and name: + return name + if isinstance(task_id, str) and task_id: + return Path(task_id).name + return result_path.parent.name + + +def _reward(result: dict[str, Any]) -> int | float | None: + return _number(_nested(result, "verifier_result", "rewards", "reward")) + + +def _trajectory_path(result_path: Path, result: dict[str, Any]) -> Path: + configured = _nested(result, "agent_result", "trajectory_path") + if isinstance(configured, str) and configured: + path = Path(configured) + if path.is_file(): + return path + relative = result_path.parent / path + if relative.is_file(): + return relative + return result_path.parent / "agent" / "trajectory.json" + + +def _observed_models( + agent_result: dict[str, Any], + trajectory: dict[str, Any], +) -> set[str]: + observed: set[str] = set() + runtime_model = agent_result.get("runtime_model_name") + if isinstance(runtime_model, str) and runtime_model: + observed.add(runtime_model) + values = _nested(trajectory, "extra", "observed_models") + if isinstance(values, list): + observed.update(str(value) for value in values if value) + model_name = _nested(trajectory, "agent", "model_name") + if isinstance(model_name, str) and model_name: + observed.add(model_name.rsplit("/", 1)[-1]) + return observed + + +def _identity_status( + *, + expected_model_id: str, + observed_models: set[str], + agent_result: dict[str, Any], + trajectory_exists: bool, +) -> str: + if not trajectory_exists: + return "trace_missing" + if not observed_models: + return "not_observed" + if observed_models == {expected_model_id} and ( + agent_result.get("canonical_model_identity") is not False + ): + return "match" + return "mismatch" + + +def _observation(step: dict[str, Any], call_id: str) -> str: + results = _nested(step, "observation", "results") + if not isinstance(results, list): + return "" + matching = [ + item + for item in results + if isinstance(item, dict) + and (not call_id or str(item.get("source_call_id") or "") == call_id) + ] + selected = matching or [item for item in results if isinstance(item, dict)] + return "\n".join(str(item.get("content") or "") for item in selected) + + +def _write_csv(path: Path, fields: tuple[str, ...], rows: list[dict[str, Any]]) -> None: + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + writer.writerows(rows) + + +def _job_directories(extracted_root: Path) -> dict[str, Path]: + directories: dict[str, Path] = {} + for manifest_path in extracted_root.rglob("run_manifest.json"): + manifest = _read_json(manifest_path) + label = str((manifest or {}).get("run_label") or "") + if label: + directories[label] = manifest_path.parent + return directories + + +def _labeled_files( + extracted_root: Path, + filename: str, + *, + label_from_parent_prefix: str | None = None, +) -> dict[str, Path]: + files: dict[str, Path] = {} + for path in extracted_root.rglob(filename): + parent_name = path.parent.name + if label_from_parent_prefix: + if not parent_name.startswith(label_from_parent_prefix): + continue + label = parent_name.removeprefix(label_from_parent_prefix) + else: + label = parent_name + if label: + files[label] = path + return files + + +def _installed_harness_version( + toolchain: dict[str, Any], + harness: str, +) -> str: + if harness == "hermes": + return str(toolchain.get("hermes_commit") or toolchain.get("hermes") or "") + key = { + "openclaw": "openclaw", + "codex": "codex", + "claude-code": "claude_code", + }.get(harness, harness) + return str(toolchain.get(key) or "") + + +def _version_status(requested: str, installed: str) -> str: + if not installed: + return "missing" + if installed == requested or requested in installed: + return "match" + return "mismatch" + + +def export_research_tables( + *, + run_index_path: Path, + extracted_root: Path, + output_dir: Path, +) -> dict[str, Any]: + run_index = _read_json(run_index_path) + if run_index is None or not isinstance(run_index.get("runs"), list): + raise ValueError(f"invalid run index: {run_index_path}") + output_dir.mkdir(parents=True, exist_ok=True) + jobs = _job_directories(extracted_root) + toolchain_paths = _labeled_files( + extracted_root, + "toolchain_manifest.json", + label_from_parent_prefix="shellbench_meta-", + ) + proxy_logs = _labeled_files(extracted_root, "proxy.log") + trace_rows: list[dict[str, Any]] = [] + turn_rows: list[dict[str, Any]] = [] + tool_rows: list[dict[str, Any]] = [] + run_rows: list[dict[str, Any]] = [] + + for entry_value in run_index["runs"]: + if not isinstance(entry_value, dict): + continue + entry = entry_value + run_label = str(entry.get("run_label") or "") + job_dir = jobs.get(run_label) + toolchain_path = toolchain_paths.get(run_label) + toolchain = _read_json(toolchain_path) if toolchain_path else {} + toolchain = toolchain or {} + harness = str(entry.get("harness") or "") + requested_harness_version = str(entry.get("harness_version") or "") + installed_harness_version = _installed_harness_version(toolchain, harness) + harness_version_status = _version_status( + requested_harness_version, + installed_harness_version, + ) + proxy_log_path = proxy_logs.get(run_label) + counters: Counter[str] = Counter() + if job_dir is not None: + result_paths = sorted( + path for path in job_dir.rglob("result.json") if path != job_dir / "result.json" + ) + else: + result_paths = [] + + for result_path in result_paths: + result = _read_json(result_path) + if result is None: + continue + task_name = _task_name(result, result_path) + agent_result = result.get("agent_result") + if not isinstance(agent_result, dict): + agent_result = {} + trajectory_path = _trajectory_path(result_path, result) + trajectory = _read_json(trajectory_path) or {} + trajectory_exists = bool(trajectory) + observed = _observed_models(agent_result, trajectory) + expected_model_id = str(entry.get("model_id") or "") + identity_status = _identity_status( + expected_model_id=expected_model_id, + observed_models=observed, + agent_result=agent_result, + trajectory_exists=trajectory_exists, + ) + counters[identity_status] += 1 + if trajectory_exists: + counters["real_trace"] += 1 + steps = trajectory.get("steps") + if not isinstance(steps, list): + steps = [] + input_tokens, cache_tokens, output_tokens = _usage(agent_result, trajectory) + cost, cost_provenance = _cost(agent_result, trajectory) + counters[ + "task_cost_exact" + if cost is not None + else "task_cost_unavailable" + ] += 1 + task_tool_count = 0 + + for turn_index, step_value in enumerate(steps): + if not isinstance(step_value, dict): + continue + step = step_value + tool_calls = step.get("tool_calls") + if not isinstance(tool_calls, list): + tool_calls = [] + task_tool_count += len(tool_calls) + turn_usage = _step_usage(step) + turn_rows.append( + { + "run_label": run_label, + "harness": entry.get("harness"), + "model_slug": entry.get("model_slug"), + "expected_model_id": expected_model_id, + "reasoning_effort": entry.get("reasoning_effort"), + "repetition": entry.get("repetition"), + "task_name": task_name, + "turn_index": turn_index, + "source": step.get("source"), + "timestamp": step.get("timestamp"), + "message_chars": len(str(step.get("message") or "")), + "reasoning_chars": len(str(step.get("reasoning_content") or "")), + "tool_call_count": len(tool_calls), + "n_input_tokens": turn_usage[0], + "n_cache_tokens": turn_usage[1], + "n_output_tokens": turn_usage[2], + "cost_usd": turn_usage[3], + "cost_provenance": turn_usage[4], + "trajectory_path": str(trajectory_path), + } + ) + for tool_index, call_value in enumerate(tool_calls): + if not isinstance(call_value, dict): + continue + call_id = str(call_value.get("tool_call_id") or "") + observation = _observation(step, call_id) + arguments = call_value.get("arguments") + tool_rows.append( + { + "run_label": run_label, + "harness": entry.get("harness"), + "model_slug": entry.get("model_slug"), + "expected_model_id": expected_model_id, + "reasoning_effort": entry.get("reasoning_effort"), + "repetition": entry.get("repetition"), + "task_name": task_name, + "turn_index": turn_index, + "tool_index": tool_index, + "tool_call_id": call_id, + "function_name": call_value.get("function_name"), + "arguments_json": json.dumps( + arguments, + ensure_ascii=True, + sort_keys=True, + ), + "observation_chars": len(observation), + "observation_excerpt": observation[:500], + "trajectory_path": str(trajectory_path), + } + ) + + trace_rows.append( + { + "run_label": run_label, + "harness": harness, + "harness_version": requested_harness_version, + "installed_harness_version": installed_harness_version, + "harness_version_status": harness_version_status, + "model_slug": entry.get("model_slug"), + "expected_model_id": expected_model_id, + "reasoning_effort": entry.get("reasoning_effort"), + "judge_model_id": entry.get("judge_model_id"), + "judge_reasoning_effort": entry.get("judge_reasoning_effort"), + "judge_identity_status": ( + "unverified_requires_proxy_request_evidence" + ), + "repetition": entry.get("repetition"), + "task_name": task_name, + "reward": _reward(result), + "result_path": str(result_path), + "trajectory_path": str(trajectory_path), + "toolchain_manifest_path": ( + str(toolchain_path) if toolchain_path else "" + ), + "proxy_log_path": str(proxy_log_path) if proxy_log_path else "", + "trajectory_status": agent_result.get("trajectory_status"), + "observed_model_ids": json.dumps(sorted(observed)), + "model_identity_status": identity_status, + "turn_count": len(steps), + "tool_call_count": task_tool_count, + "n_input_tokens": input_tokens, + "n_cache_tokens": cache_tokens, + "n_output_tokens": output_tokens, + "cost_usd": cost, + "cost_provenance": cost_provenance, + } + ) + + result_count = len(result_paths) + passed = result_count > 0 and ( + counters["match"] == result_count + and counters["real_trace"] == result_count + ) + run_rows.append( + { + "run_label": run_label, + "harness": harness, + "harness_version": requested_harness_version, + "installed_harness_version": installed_harness_version, + "harness_version_status": harness_version_status, + "model_slug": entry.get("model_slug"), + "expected_model_id": entry.get("model_id"), + "reasoning_effort": entry.get("reasoning_effort"), + "judge_model_id": entry.get("judge_model_id"), + "judge_reasoning_effort": entry.get("judge_reasoning_effort"), + "judge_identity_status": ( + "unverified_requires_proxy_request_evidence" + ), + "repetition": entry.get("repetition"), + "expected_task_count": entry.get("expected_task_count"), + "result_count": result_count, + "real_trace_count": counters["real_trace"], + "identity_match_count": counters["match"], + "identity_mismatch_count": counters["mismatch"], + "identity_not_observed_count": counters["not_observed"], + "trace_missing_count": counters["trace_missing"], + "model_identity_audit_passed": passed, + "toolchain_manifest_path": ( + str(toolchain_path) if toolchain_path else "" + ), + "proxy_log_path": str(proxy_log_path) if proxy_log_path else "", + "task_cost_exact_count": counters["task_cost_exact"], + "task_cost_unavailable_count": counters["task_cost_unavailable"], + } + ) + + _write_csv(output_dir / "trace_inventory.csv", TRACE_FIELDS, trace_rows) + _write_csv(output_dir / "turn_usage.csv", TURN_FIELDS, turn_rows) + _write_csv(output_dir / "tool_calls.csv", TOOL_FIELDS, tool_rows) + _write_csv(output_dir / "model_identity_audit.csv", RUN_AUDIT_FIELDS, run_rows) + summary = { + "run_count": len(run_rows), + "task_result_count": len(trace_rows), + "turn_count": len(turn_rows), + "tool_call_count": len(tool_rows), + "identity_audit_pass_count": sum( + row["model_identity_audit_passed"] is True for row in run_rows + ), + "identity_audit_fail_count": sum( + row["model_identity_audit_passed"] is not True for row in run_rows + ), + "exact_task_cost_count": sum( + row["cost_provenance"] in {"exact_harness", "exact_trace"} + for row in trace_rows + ), + "unavailable_task_cost_count": sum( + row["cost_usd"] is None for row in trace_rows + ), + "outputs": { + "trace_inventory": "trace_inventory.csv", + "turn_usage": "turn_usage.csv", + "tool_calls": "tool_calls.csv", + "model_identity_audit": "model_identity_audit.csv", + }, + } + (output_dir / "research_audit.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return summary + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--run-index", type=Path, required=True) + parser.add_argument("--extracted-root", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args(argv) + + +def main() -> None: + args = parse_args() + summary = export_research_tables( + run_index_path=args.run_index, + extracted_root=args.extracted_root, + output_dir=args.output_dir, + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_native_eval_research_audit.py b/tests/test_native_eval_research_audit.py new file mode 100644 index 0000000..9fb8e59 --- /dev/null +++ b/tests/test_native_eval_research_audit.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from scripts.native_eval.research_audit import export_research_tables + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def test_research_audit_exports_identity_turn_tool_and_usage_tables( + tmp_path: Path, +) -> None: + run_label = "openclaw-gpt56-sol-high-full-1-r1-20260729" + run_index = tmp_path / "run-index.json" + extracted = tmp_path / "extracted" + output = tmp_path / "analysis" + job_dir = extracted / run_label + trial_dir = job_dir / "task__abc" + trajectory_path = trial_dir / "agent" / "trajectory.json" + _write_json( + run_index, + { + "runs": [ + { + "run_label": run_label, + "harness": "openclaw", + "harness_version": "2026.7.1-2", + "model_slug": "gpt56-sol", + "model_id": "gpt-5.6-sol", + "reasoning_effort": "high", + "judge_model_id": "gpt-5.6-sol", + "judge_reasoning_effort": "high", + "repetition": 1, + "expected_task_count": 1, + } + ] + }, + ) + _write_json(job_dir / "run_manifest.json", {"run_label": run_label}) + _write_json( + extracted / f"shellbench_meta-{run_label}" / "toolchain_manifest.json", + {"openclaw": "openclaw 2026.7.1-2"}, + ) + (extracted / "proxy" / run_label).mkdir(parents=True) + (extracted / "proxy" / run_label / "proxy.log").write_text( + "proxy output\n", + encoding="utf-8", + ) + _write_json( + trial_dir / "result.json", + { + "task_id": {"path": "/tasks/example-task"}, + "verifier_result": {"rewards": {"reward": 1}}, + "agent_result": { + "trajectory_status": "real", + "runtime_model_name": "gpt-5.6-sol", + "canonical_model_identity": True, + "n_input_tokens": 120, + "n_cache_tokens": 20, + "n_output_tokens": 30, + "cost_usd": 0.25, + }, + }, + ) + _write_json( + trajectory_path, + { + "agent": { + "name": "openclaw", + "version": "2026.7.1-2", + "model_name": "openai/gpt-5.6-sol", + }, + "steps": [ + { + "source": "agent", + "message": "", + "usage": { + "input_tokens": 12, + "output_tokens": 3, + "cost_usd": 0.01, + }, + "tool_calls": [ + { + "tool_call_id": "call-1", + "function_name": "shell", + "arguments": {"command": "pwd"}, + } + ], + "observation": { + "results": [ + { + "source_call_id": "call-1", + "content": "/workspace", + } + ] + }, + } + ], + "final_metrics": { + "total_prompt_tokens": 120, + "total_cached_tokens": 20, + "total_completion_tokens": 30, + "total_cost_usd": 0.25, + }, + "extra": {"observed_models": ["gpt-5.6-sol"]}, + }, + ) + + summary = export_research_tables( + run_index_path=run_index, + extracted_root=extracted, + output_dir=output, + ) + + assert summary["identity_audit_pass_count"] == 1 + assert summary["task_result_count"] == 1 + assert summary["turn_count"] == 1 + assert summary["tool_call_count"] == 1 + with (output / "trace_inventory.csv").open(newline="", encoding="utf-8") as handle: + trace_row = next(csv.DictReader(handle)) + assert trace_row["model_identity_status"] == "match" + assert trace_row["harness_version_status"] == "match" + assert trace_row["installed_harness_version"] == "openclaw 2026.7.1-2" + assert trace_row["judge_identity_status"] == ( + "unverified_requires_proxy_request_evidence" + ) + assert trace_row["cost_provenance"] == "exact_harness" + with (output / "turn_usage.csv").open(newline="", encoding="utf-8") as handle: + turn_row = next(csv.DictReader(handle)) + assert turn_row["n_input_tokens"] == "12" + assert turn_row["cost_provenance"] == "exact_trace_turn" + with (output / "tool_calls.csv").open(newline="", encoding="utf-8") as handle: + tool_row = next(csv.DictReader(handle)) + assert tool_row["function_name"] == "shell" + assert tool_row["observation_excerpt"] == "/workspace" + + +def test_research_audit_fails_identity_when_trace_is_missing(tmp_path: Path) -> None: + run_label = "hermes-gpt56-sol-low-full-1-r1-20260729" + run_index = tmp_path / "run-index.json" + extracted = tmp_path / "extracted" + job_dir = extracted / run_label + _write_json( + run_index, + { + "runs": [ + { + "run_label": run_label, + "harness": "hermes", + "model_slug": "gpt56-sol", + "model_id": "gpt-5.6-sol", + "reasoning_effort": "low", + "repetition": 1, + "expected_task_count": 1, + } + ] + }, + ) + _write_json(job_dir / "run_manifest.json", {"run_label": run_label}) + _write_json( + job_dir / "task__abc" / "result.json", + { + "task_id": {"path": "/tasks/example-task"}, + "agent_result": {"trajectory_status": "unavailable"}, + }, + ) + + summary = export_research_tables( + run_index_path=run_index, + extracted_root=extracted, + output_dir=tmp_path / "analysis", + ) + + assert summary["identity_audit_pass_count"] == 0 + assert summary["identity_audit_fail_count"] == 1 From 64f589019d61c80f926aae29da6a72f41d444153 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 12:03:51 +0200 Subject: [PATCH 3/3] fix(eval): gate full runs behind retained r0 --- .../shellbench-research-runbook/SKILL.md | 46 +++++- .../references/runbook.md | 124 +++++++++++---- CHANGELOG.md | 6 +- scripts/native_eval/fleet.py | 50 +++++- scripts/native_eval/models.py | 9 +- scripts/native_eval/plan.py | 60 +++++++- scripts/native_eval/research_audit.py | 16 ++ scripts/native_eval/run_job.py | 20 ++- tests/test_native_eval_fleet.py | 58 ++++++- tests/test_native_eval_research_audit.py | 2 + tests/test_native_eval_runner.py | 143 ++++++++++++++++++ 11 files changed, 476 insertions(+), 58 deletions(-) diff --git a/.agents/skills/shellbench-research-runbook/SKILL.md b/.agents/skills/shellbench-research-runbook/SKILL.md index 3f89273..2bad184 100644 --- a/.agents/skills/shellbench-research-runbook/SKILL.md +++ b/.agents/skills/shellbench-research-runbook/SKILL.md @@ -17,11 +17,13 @@ schema, and recovery rules. trials on the operator laptop. 2. Pin one public-task commit, runner commit or patch hash, provider model ID, harness version, reasoning level, and judge route for the whole campaign. -3. Run three-task smokes for every distinct harness, model, and reasoning route. - Do not start full-suite jobs until model identity, real traces, tools, usage, - judge routing, and artifact export pass. -4. Qualify with independent repetitions `r1` through `r3`. After a clean audit, - add `r4` through `r6`; the research result is six total repetitions. +3. Run one `r0` qualification for every distinct harness and model-family + route, using exactly ten pinned representative tasks. Do not start full-suite + jobs until model identity, real traces, tools, usage, judge routing, and + artifact export pass. +4. Retain and audit every `r0`, but force it out of leaderboard scoring. Qualify + with independent full-suite repetitions `r1` through `r3`. After a clean + audit, add `r4` through `r6`; the research result is six total repetitions. 5. Run every provider-supported non-maximum reasoning level. Never label a reasoning level as tested unless the route applies it and the trace or proxy evidence proves it. Record unsupported levels instead of fabricating them. @@ -53,9 +55,39 @@ python -m scripts.native_eval.plan \ --repetitions 3 ``` -Use repeatable `--harness` and `--model` filters for smoke or phased plans. +Use repeatable `--harness` and `--model` filters for r0 or phased plans. Set `--repetitions 6` only after the first three repetitions pass qualification. +Generate each family/harness r0 separately: + +```sh +python -m scripts.native_eval.plan \ + --tasks-root "$TASKS_ROOT" \ + --output "$CAMPAIGN/manifests/r0-openclaw-gpt56.json" \ + --public-tasks-commit "$PUBLIC_TASKS_COMMIT" \ + --run-date "$RUN_DATE" \ + --phase r0 \ + --qualification-family gpt-5.6 \ + --harness openclaw \ + --model gpt56-sol \ + --reasoning-effort high \ + --judge-model-id gpt-5.6-sol \ + --judge-reasoning-effort high \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" \ + --task "" +``` + +The planner marks r0 `leaderboard_eligible=false` with exclusion reason +`r0_non_scoring_qualification`. + After extraction, produce the research tables and strict identity report: ```sh @@ -70,7 +102,7 @@ does not invent prices: missing exact spend remains explicitly unavailable. ## Stop conditions -- Stop a route after any smoke contains the wrong or multiple model IDs. +- Stop a route after any r0 contains the wrong or multiple model IDs. - Stop when the judge's observed provider model cannot be proven. - Stop when a harness version differs from its campaign pin. - Preserve and exclude infra-dominated runs, then rerun the same repetition diff --git a/.agents/skills/shellbench-research-runbook/references/runbook.md b/.agents/skills/shellbench-research-runbook/references/runbook.md index 4526d67..dd8a1f0 100644 --- a/.agents/skills/shellbench-research-runbook/references/runbook.md +++ b/.agents/skills/shellbench-research-runbook/references/runbook.md @@ -3,11 +3,11 @@ ## 2026-07-29 Revision Appendix This replaces the original full-matrix prompt with stronger research gates: -three-task route smokes, verified private S3 retention, strict observed-model -audits, requested-versus-installed harness versions, n=3 qualification followed -by n=6 total repetitions, every provider-supported non-maximum reasoning level, -`gpt-5.6-sol` at `high` as judge, and task/turn/tool/token/cost exports with -explicit evidence provenance. +ten-task non-scoring `r0` family/harness qualification runs, verified private S3 +retention, strict observed-model audits, requested-versus-installed harness +versions, n=3 qualification followed by n=6 total repetitions, every +provider-supported non-maximum reasoning level, `gpt-5.6-sol` at `high` as +judge, and task/turn/tool/token/cost exports with explicit evidence provenance. ## Objective @@ -17,14 +17,16 @@ reasoning levels. Preserve enough evidence to reproduce every score and audit which model, harness build, tools, tokens, costs, judge, task commit, and runner code produced it. -The campaign progresses through three gates: +The campaign progresses through four gates: -1. route smoke: three representative tasks, one repetition -2. qualification: full suite, independent repetitions `r1` through `r3` -3. research completion: add `r4` through `r6` after qualification is clean +1. route qualification: ten representative tasks at `r0` +2. initial research: full suite, independent repetitions `r1` through `r3` +3. audit: verify coverage, model identity, versions, traces, usage, and infra +4. research completion: add `r4` through `r6` only after the audit is clean `n=6` means six independent jobs and writable environments. It does not mean -rerunning one job directory or replaying saved agent state. +rerunning one job directory or replaying saved agent state. `r0` is not part of +`n=6`. ## Scope @@ -78,7 +80,7 @@ Before every campaign: 1. compare this table with the code pins 2. record requested pins in the campaign manifest 3. record installed versions from `/opt/shellbench-native/manifest.json` -4. fail smoke when requested and installed versions differ +4. fail r0 when requested and installed versions differ Do not update a pin during a campaign. Start a new campaign ID when a harness, runner, task, provider route, or judge version changes. @@ -104,10 +106,10 @@ openclaw-gpt56-sol-low-full-115-r1-20260729 hermes-gpt56-sol-high-full-115-r6-20260729 ``` -Smokes use: +Family/harness qualification uses: ```text ----smoke--r1- +---smoke-10-r0- ``` Retries append a suffix and never replace the original: @@ -181,31 +183,72 @@ Record: - requested reasoning and whether the adapter can prove it - judge provider ID, reasoning, proxy alias, and observed identity evidence -## Phase 1: Route Smokes +## Phase 1: Ten-Task r0 Qualification -Select three tasks from the pinned suite: +Select exactly ten tasks from the pinned suite. The set should cover: -1. a simple shell or filesystem task -2. a browser, app, or multi-container task -3. a judge-backed or semantically verified task +1. simple shell and filesystem work +2. code editing and test execution +3. browser, app, or multi-container work +4. long-running or stateful tool use +5. judge-backed or semantically verified work +6. representative easy, medium, and difficult tasks Record task names and checksums. Do not hard-code task names in the runbook because the public suite changes. -Run each distinct harness, model, and reasoning route with: +Run one r0 for every distinct harness and model-family route with: - one fresh beast or isolated job -- repetition `1` +- repetition `0` +- exactly one harness and one representative provider model +- a recorded `qualification_family` - concurrency `1` or `2` - the same proxy and judge configuration intended for the full campaign - checkpointing enabled immediately -Smokes may run in parallel across beasts. Do not increase per-smoke concurrency; -the purpose is routing and evidence validation, not throughput. +r0 jobs may run in parallel across beasts. Do not increase per-r0 concurrency; +the purpose is routing and evidence validation, not throughput. Family +qualification proves the shared harness and routing path; every exact provider +model ID is still audited again during `r1` through `r3`. -For every smoke, verify: +Treat a different reasoning transport, proxy parameter path, or provider adapter +as a different route and give it its own r0. Do not assume a high-reasoning r0 +proves low or medium when those settings travel through different code. -- all three task results exist +Generate one r0 plan per harness/family: + +```sh +python -m scripts.native_eval.plan \ + --tasks-root "$TASKS_ROOT" \ + --output "$CAMPAIGN/manifests/r0-$HARNESS-$FAMILY.json" \ + --public-tasks-commit "$PUBLIC_TASKS_COMMIT" \ + --run-date "$RUN_DATE" \ + --phase r0 \ + --qualification-family "$FAMILY" \ + --harness "$HARNESS" \ + --model "$REPRESENTATIVE_MODEL_SLUG" \ + --reasoning-effort "$REASONING" \ + --judge-model-id gpt-5.6-sol \ + --judge-reasoning-effort high \ + --task "$TASK_01" \ + --task "$TASK_02" \ + --task "$TASK_03" \ + --task "$TASK_04" \ + --task "$TASK_05" \ + --task "$TASK_06" \ + --task "$TASK_07" \ + --task "$TASK_08" \ + --task "$TASK_09" \ + --task "$TASK_10" +``` + +The planner enforces ten tasks, repetition zero, one harness, one model, and +automatic leaderboard exclusion. + +For every r0, verify: + +- all ten task results exist - `trajectory_status` is `real` for supported harnesses - raw harness events and normalized `trajectory.json` are both retained - observed agent model IDs equal exactly the requested provider model ID @@ -220,14 +263,18 @@ For every smoke, verify: - verified archives upload to S3 and can be read back or headed The agent trace alone cannot prove judge identity. Preserve proxy/provider -request logs and audit the dedicated judge alias during smoke. +request logs and audit the dedicated judge alias during r0. Any wrong, mixed, or unobserved model ID blocks that route. Fix the route and -repeat the smoke under a new label. +repeat r0 under a new suffixed label. -## Phase 2: Qualification At n=3 +Retain every r0 checkpoint, final archive, trace, log, and audit row. Discard r0 +only from scoring: its manifest must set `leaderboard_eligible=false` and +`exclusion_reason=r0_non_scoring_qualification`. -Generate one plan per reasoning level so labels remain unique: +## Phase 2: Initial Full-Suite Runs At n=3 + +After every required r0 passes, generate one full plan per reasoning level: ```sh python -m scripts.native_eval.plan \ @@ -261,7 +308,22 @@ browser and app-heavy routes at task concurrency `16`; lower to `8` after startup pressure, or raise a proven stable route to `32`. Do not use `96+` except as a named infrastructure stress experiment. -## Phase 3: Research Completion At n=6 +## Phase 3: Audit r1 Through r3 + +Do not schedule `r4` through `r6` until every exact harness/model/reasoning route +has three complete full-suite runs and the audit confirms: + +- complete task coverage +- exact requested agent model identity +- requested reasoning evidence +- installed harness versions match campaign pins +- judge routing is proven +- traces and provider/proxy logs are retained locally and in S3 +- infrastructure failures are low enough for a fair comparison + +Fix or rerun only failed routes and repetitions, then repeat the audit. + +## Phase 4: Research Completion At n=6 Only after all three qualification repetitions are complete and clean: @@ -360,6 +422,9 @@ The command writes: - `tool_calls.csv`: one row per tool call - `research_audit.json`: counts and audit status +The tables retain r0 rows with `phase=r0` and +`leaderboard_eligible=false`. Aggregate score outputs must exclude them. + Identity passes only when every recovered task trace observes exactly the requested model ID. Missing traces and missing observed identity fail the audit; they are not treated as neutral. @@ -448,6 +513,7 @@ runs-full-YYYYMMDD/ The audit note must state: +- retained r0 qualification status by harness and model family - clean, excluded, missing, and rerun-required repetitions - full task coverage against the pinned task commit - requested and observed model IDs diff --git a/CHANGELOG.md b/CHANGELOG.md index cb33aeb..893edea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,6 @@ - Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog. - Report repair-overlay task provenance and original-versus-repaired score sensitivity when the original job is available. -- Add a research campaign skill with smoke gates, n=3 through n=6 execution, - private trace retention, strict model-identity checks, pinned harness - versions, and task/turn/tool/token/cost analysis exports. +- Add a research campaign skill with retained non-scoring ten-task r0 gates, + n=3 through n=6 execution, private trace retention, strict model-identity + checks, pinned harness versions, and task/turn/tool/token/cost exports. diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index c7af503..433d775 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -387,6 +387,7 @@ def _validate_plan(self) -> None: for entry in self._store.all_entries(): run = self._run_spec(entry) task_names = entry.get("task_names") + phase = str(entry.get("phase") or "full") if task_names is None: expected = suite_expected elif not isinstance(task_names, list) or not all( @@ -399,10 +400,27 @@ def _validate_plan(self) -> None: raise FleetError(f"{run.run_label} task_names contains duplicates") else: expected = len(task_names) - if not entry.get("rerun_of_canonical_run"): + if phase == "r0": + if run.repetition != 0: + raise FleetError(f"{run.run_label} r0 must use repetition 0") + if len(task_names) != 10: + raise FleetError(f"{run.run_label} r0 must select 10 tasks") + if entry.get("leaderboard_eligible") is not False: + raise FleetError( + f"{run.run_label} r0 must be leaderboard-ineligible" + ) + if not entry.get("qualification_family"): + raise FleetError( + f"{run.run_label} r0 lacks qualification_family" + ) + elif not entry.get("rerun_of_canonical_run"): raise FleetError( f"{run.run_label} task subset lacks rerun_of_canonical_run" ) + if phase == "r0" and task_names is None: + raise FleetError(f"{run.run_label} r0 lacks task_names") + if phase != "r0" and run.repetition == 0: + raise FleetError(f"{run.run_label} repetition 0 requires phase r0") if run.expected_task_count != expected: raise FleetError( f"{run.run_label} expects {run.expected_task_count} tasks, " @@ -933,9 +951,14 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: execution_mode=$3 reasoning_effort=$4 judge_reasoning_effort=$5 -parity_validated=$6 -parity_validation_json=$7 -shift 7 +qualification_family=$6 +run_phase=$7 +leaderboard_eligible=$8 +exclusion_reason=$9 +shift 9 +parity_validated=$1 +parity_validation_json=$2 +shift 2 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -952,6 +975,10 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_EXECUTION_MODE=$execution_mode" \ "SHELLBENCH_REASONING_EFFORT=$reasoning_effort" \ "SHELLBENCH_JUDGE_REASONING_EFFORT=$judge_reasoning_effort" \ + "SHELLBENCH_QUALIFICATION_FAMILY=$qualification_family" \ + "SHELLBENCH_RUN_PHASE=$run_phase" \ + "SHELLBENCH_LEADERBOARD_ELIGIBLE=$leaderboard_eligible" \ + "SHELLBENCH_EXCLUSION_REASON=$exclusion_reason" \ "SHELLBENCH_PARITY_VALIDATED=$parity_validated" \ "SHELLBENCH_PARITY_VALIDATION_JSON=$parity_validation_json" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ @@ -985,6 +1012,12 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: judge_reasoning_effort = str( entry.get("judge_reasoning_effort") or reasoning_effort ) + qualification_family = str(entry.get("qualification_family") or "") + run_phase = str(entry.get("phase") or "full") + leaderboard_eligible = ( + "false" if entry.get("leaderboard_eligible") is False else "" + ) + exclusion_reason = str(entry.get("exclusion_reason") or "") parity_validation = "" if (run.harness, run.model_slug) in self.config.parity_validated_routes: parity_validation = json.dumps( @@ -1019,6 +1052,10 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: self.config.execution_mode, reasoning_effort, judge_reasoning_effort, + qualification_family, + run_phase, + leaderboard_eligible, + exclusion_reason, str(self.config.parity_validated).lower(), parity_validation, *args, @@ -1426,9 +1463,12 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "run_label": label, "reasoning_effort": entry.get("reasoning_effort"), "judge_reasoning_effort": entry.get("judge_reasoning_effort"), + "phase": entry.get("phase"), + "qualification_family": entry.get("qualification_family"), "attempt": next_attempt, "status": "planned", - "leaderboard_eligible": None, + "leaderboard_eligible": entry.get("leaderboard_eligible"), + "exclusion_reason": entry.get("exclusion_reason"), "rerun_of": root_label, "lease": None, "artifacts": [], diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index c0204ba..cbe08be 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -130,13 +130,16 @@ def build_matrix_plan( models: Iterable[ModelSpec] = MODELS, repetitions: Iterable[int] = (1, 2, 3), reasoning_effort: str | None = None, + run_kind: str = "full", ) -> list[RunSpec]: stamp = run_date or date.today().strftime("%Y%m%d") repetition_values = tuple(repetitions) - if not repetition_values or any(value < 1 for value in repetition_values): - raise ValueError("repetitions must contain positive integers") + if not repetition_values or any(value < 0 for value in repetition_values): + raise ValueError("repetitions must contain non-negative integers") if len(set(repetition_values)) != len(repetition_values): raise ValueError("repetitions must not contain duplicates") + if not run_kind or "-" in run_kind: + raise ValueError("run_kind must be a non-empty label segment") reasoning_slug = reasoning_effort.replace("_", "-") if reasoning_effort else None plan: list[RunSpec] = [] for harness in harnesses: @@ -144,7 +147,7 @@ def build_matrix_plan( for repetition in repetition_values: reasoning_label = f"-{reasoning_slug}" if reasoning_slug else "" label = ( - f"{harness.name}-{model.slug}{reasoning_label}-full-" + f"{harness.name}-{model.slug}{reasoning_label}-{run_kind}-" f"{expected_task_count}-r{repetition}-{stamp}" ) plan.append( diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index fca0bd9..372aec0 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -28,6 +28,9 @@ def write_run_index( repetitions: int = 3, harness_names: Sequence[str] | None = None, model_slugs: Sequence[str] | None = None, + phase: str = "full", + task_names: Sequence[str] | None = None, + qualification_family: str | None = None, ) -> list[dict[str, object]]: tasks = validate_suite(tasks_root) selected_harnesses = tuple( @@ -44,13 +47,41 @@ def write_run_index( raise ValueError("no harnesses selected") if not selected_models: raise ValueError("no models selected") + selected_task_names = list(task_names or []) + suite_task_names = {task.name for task in tasks} if selected_task_names else set() + unknown_tasks = set(selected_task_names) - suite_task_names + if unknown_tasks: + raise ValueError(f"unknown task names: {', '.join(sorted(unknown_tasks))}") + if len(set(selected_task_names)) != len(selected_task_names): + raise ValueError("task names must not contain duplicates") + if phase == "r0": + if len(selected_harnesses) != 1 or len(selected_models) != 1: + raise ValueError("r0 plans require exactly one harness and one model") + if len(selected_task_names) != 10: + raise ValueError("r0 plans require exactly 10 named tasks") + if not qualification_family: + raise ValueError("r0 plans require qualification_family") + planned_repetitions = (0,) + expected_task_count = len(selected_task_names) + run_kind = "smoke" + elif phase == "full": + if selected_task_names: + raise ValueError("full plans do not accept task subsets") + if qualification_family: + raise ValueError("qualification_family is only valid for r0 plans") + planned_repetitions = tuple(range(1, repetitions + 1)) + expected_task_count = len(tasks) + run_kind = "full" + else: + raise ValueError(f"unsupported phase: {phase}") runs = build_matrix_plan( - len(tasks), + expected_task_count, run_date=run_date, harnesses=selected_harnesses, models=selected_models, - repetitions=range(1, repetitions + 1), + repetitions=planned_repetitions, reasoning_effort=reasoning_effort, + run_kind=run_kind, ) entries = [ { @@ -58,9 +89,15 @@ def write_run_index( "reasoning_effort": reasoning_effort, "judge_model_id": judge_model_id, "judge_reasoning_effort": judge_reasoning_effort, + "phase": phase, + "qualification_family": qualification_family, + "task_names": selected_task_names or None, "attempt": 0, "status": "planned", - "leaderboard_eligible": None, + "leaderboard_eligible": False if phase == "r0" else None, + "exclusion_reason": ( + "r0_non_scoring_qualification" if phase == "r0" else None + ), "rerun_of": None, "lease": None, "artifacts": [], @@ -75,7 +112,11 @@ def write_run_index( "task_suite_path": "combined tasks/tasks", "expected_task_count": len(tasks), "planned_run_count": len(entries), - "repetition_count": repetitions, + "repetition_count": len(planned_repetitions), + "planned_repetitions": list(planned_repetitions), + "phase": phase, + "qualification_family": qualification_family, + "qualification_task_names": selected_task_names, "reasoning_effort": reasoning_effort, "judge_model_id": judge_model_id, "judge_reasoning_effort": judge_reasoning_effort, @@ -104,6 +145,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--judge-model-id", default="gpt-5.6-sol") parser.add_argument("--repetitions", type=_positive_int, default=3) + parser.add_argument("--phase", choices=("r0", "full"), default="full") + parser.add_argument("--qualification-family") + parser.add_argument( + "--task", + action="append", + dest="task_names", + help="Select one r0 qualification task. Repeat exactly ten times.", + ) parser.add_argument( "--harness", action="append", @@ -134,6 +183,9 @@ def main() -> None: repetitions=args.repetitions, harness_names=args.harness_names, model_slugs=args.model_slugs, + phase=args.phase, + task_names=args.task_names, + qualification_family=args.qualification_family, ) print(f"wrote {len(entries)} planned runs to {args.output}") diff --git a/scripts/native_eval/research_audit.py b/scripts/native_eval/research_audit.py index 2ab8e34..381df62 100644 --- a/scripts/native_eval/research_audit.py +++ b/scripts/native_eval/research_audit.py @@ -24,6 +24,9 @@ "judge_reasoning_effort", "judge_identity_status", "repetition", + "phase", + "qualification_family", + "leaderboard_eligible", "task_name", "reward", "result_path", @@ -95,6 +98,9 @@ "judge_reasoning_effort", "judge_identity_status", "repetition", + "phase", + "qualification_family", + "leaderboard_eligible", "expected_task_count", "result_count", "real_trace_count", @@ -515,6 +521,9 @@ def export_research_tables( "unverified_requires_proxy_request_evidence" ), "repetition": entry.get("repetition"), + "phase": entry.get("phase") or "full", + "qualification_family": entry.get("qualification_family"), + "leaderboard_eligible": entry.get("leaderboard_eligible"), "task_name": task_name, "reward": _reward(result), "result_path": str(result_path), @@ -557,6 +566,9 @@ def export_research_tables( "unverified_requires_proxy_request_evidence" ), "repetition": entry.get("repetition"), + "phase": entry.get("phase") or "full", + "qualification_family": entry.get("qualification_family"), + "leaderboard_eligible": entry.get("leaderboard_eligible"), "expected_task_count": entry.get("expected_task_count"), "result_count": result_count, "real_trace_count": counters["real_trace"], @@ -589,6 +601,10 @@ def export_research_tables( "identity_audit_fail_count": sum( row["model_identity_audit_passed"] is not True for row in run_rows ), + "r0_run_count": sum(row["phase"] == "r0" for row in run_rows), + "scoring_run_count": sum( + row["leaderboard_eligible"] is not False for row in run_rows + ), "exact_task_cost_count": sum( row["cost_provenance"] in {"exact_harness", "exact_trace"} for row in trace_rows diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index d12935f..04549c8 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -216,6 +216,11 @@ def _run_manifest( "model_provider": run.provider, "proxy_model_name": run.proxy_model_name, "repetition": run.repetition, + "phase": os.environ.get("SHELLBENCH_RUN_PHASE", "full"), + "qualification_family": os.environ.get( + "SHELLBENCH_QUALIFICATION_FAMILY" + ) + or None, "task_suite": task_suite_path, "task_suite_root": str(tasks_root.resolve()), "expected_task_count": run.expected_task_count, @@ -259,6 +264,13 @@ def _run_manifest( "judge_reasoning_effort": os.environ.get( "SHELLBENCH_JUDGE_REASONING_EFFORT" ), + "leaderboard_eligible": ( + False + if os.environ.get("SHELLBENCH_LEADERBOARD_ELIGIBLE", "").lower() + == "false" + else None + ), + "exclusion_reason": os.environ.get("SHELLBENCH_EXCLUSION_REASON") or None, "runner_commit": _git_commit(), "runner_patch_hash": _runner_patch_hash(), "public_tasks_commit": public_tasks_commit, @@ -350,10 +362,10 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: ) -def _positive_int(value: str) -> int: +def _non_negative_int(value: str) -> int: parsed = int(value) - if parsed < 1: - raise argparse.ArgumentTypeError("must be at least 1") + if parsed < 0: + raise argparse.ArgumentTypeError("must be at least 0") return parsed @@ -368,7 +380,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--model-id") parser.add_argument("--model-provider") parser.add_argument("--proxy-model-name") - parser.add_argument("--repetition", type=_positive_int, required=True) + parser.add_argument("--repetition", type=_non_negative_int, required=True) parser.add_argument("--expected-task-count", type=int, required=True) parser.add_argument("--public-tasks-commit", required=True) parser.add_argument("--task-suite-path", required=True) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index d26aaa0..b9855ff 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -28,6 +28,7 @@ def _run_spec( *, expected_task_count: int = 2, model_slug: str = "gpt55", + repetition: int = 1, ) -> RunSpec: return RunSpec( run_label=label, @@ -37,7 +38,7 @@ def _run_spec( model_id=f"provider/{model_slug}", provider="anthropic" if model_slug == "fable5" else "openai", proxy_model_name=f"sb-{model_slug}", - repetition=1, + repetition=repetition, expected_task_count=expected_task_count, run_date="20260727", ) @@ -304,6 +305,10 @@ def __init__( self.dispatch_concurrency: dict[str, int] = {} self.dispatch_arguments: dict[str, list[str]] = {} self.dispatch_parity_validation: dict[str, str] = {} + self.dispatch_qualification_family: dict[str, str] = {} + self.dispatch_phase: dict[str, str] = {} + self.dispatch_leaderboard_eligible: dict[str, str] = {} + self.dispatch_exclusion_reason: dict[str, str] = {} self.stops: list[str] = [] self._lock = threading.Lock() self._dispatch_condition = threading.Condition(self._lock) @@ -429,8 +434,12 @@ def run( ) remote_command = shlex.split(argv[-1]) dispatch_marker = remote_command.index("fleet-dispatch") - parity_validation = remote_command[dispatch_marker + 16] - remote_run_args = remote_command[dispatch_marker + 17 :] + qualification_family = remote_command[dispatch_marker + 15] + run_phase = remote_command[dispatch_marker + 16] + leaderboard_eligible = remote_command[dispatch_marker + 17] + exclusion_reason = remote_command[dispatch_marker + 18] + parity_validation = remote_command[dispatch_marker + 20] + remote_run_args = remote_command[dispatch_marker + 21 :] with self._dispatch_condition: attempt = self.dispatch_attempts.get(label, 0) + 1 self.dispatch_attempts[label] = attempt @@ -444,6 +453,10 @@ def run( self.dispatch_concurrency[label] = int(remote_run_args[10]) self.dispatch_arguments[label] = remote_run_args self.dispatch_parity_validation[label] = parity_validation + self.dispatch_qualification_family[label] = qualification_family + self.dispatch_phase[label] = run_phase + self.dispatch_leaderboard_eligible[label] = leaderboard_eligible + self.dispatch_exclusion_reason[label] = exclusion_reason self.events.append(("dispatch", label)) self.remote_states[label] = "running" self._dispatch_condition.notify_all() @@ -641,6 +654,45 @@ def test_controller_rejects_task_subset_without_canonical_parent( FleetController(config, executor=executor).run() +def test_controller_accepts_non_scoring_ten_task_r0( + tmp_path: Path, +) -> None: + label = "openclaw-gpt56-sol-high-smoke-10-r0-20260729" + run = _planned( + _run_spec( + label, + expected_task_count=10, + model_slug="gpt56-sol", + repetition=0, + ) + ) + run.update( + { + "phase": "r0", + "qualification_family": "gpt-5.6", + "task_names": [f"task-{index}" for index in range(10)], + "leaderboard_eligible": False, + "exclusion_reason": "r0_non_scoring_qualification", + } + ) + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run], expected_task_count=116) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 10}) + + assert FleetController(config, executor=executor).run() == 0 + + assert executor.dispatch_qualification_family[label] == "gpt-5.6" + assert executor.dispatch_phase[label] == "r0" + assert executor.dispatch_leaderboard_eligible[label] == "false" + assert executor.dispatch_exclusion_reason[label] == ( + "r0_non_scoring_qualification" + ) + assert executor.dispatch_arguments[label][16:] == [ + f"task-{index}" for index in range(10) + ] + + def test_capacity_warmup_retries_same_run_without_recovery_churn( tmp_path: Path, ) -> None: diff --git a/tests/test_native_eval_research_audit.py b/tests/test_native_eval_research_audit.py index 9fb8e59..8e37557 100644 --- a/tests/test_native_eval_research_audit.py +++ b/tests/test_native_eval_research_audit.py @@ -36,6 +36,7 @@ def test_research_audit_exports_identity_turn_tool_and_usage_tables( "judge_model_id": "gpt-5.6-sol", "judge_reasoning_effort": "high", "repetition": 1, + "phase": "full", "expected_task_count": 1, } ] @@ -129,6 +130,7 @@ def test_research_audit_exports_identity_turn_tool_and_usage_tables( assert trace_row["judge_identity_status"] == ( "unverified_requires_proxy_request_evidence" ) + assert trace_row["phase"] == "full" assert trace_row["cost_provenance"] == "exact_harness" with (output / "turn_usage.csv").open(newline="", encoding="utf-8") as handle: turn_row = next(csv.DictReader(handle)) diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 10cea4d..0e14fea 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -100,6 +100,45 @@ def test_run_index_supports_six_repetitions_and_filters( assert all("-medium-full-" in str(entry["run_label"]) for entry in entries) +def test_run_index_builds_non_scoring_ten_task_r0( + tmp_path: Path, + monkeypatch, +) -> None: + tasks = [SimpleNamespace(name=f"task-{index}") for index in range(12)] + monkeypatch.setattr(native_plan, "validate_suite", lambda _root: tasks) + selected = [task.name for task in tasks[:10]] + + entries = native_plan.write_run_index( + tasks_root=tmp_path, + output=tmp_path / "run-index.json", + public_tasks_commit="tasks-commit", + run_date="20260729", + reasoning_effort="high", + judge_reasoning_effort="high", + harness_names=["openclaw"], + model_slugs=["gpt56-sol"], + phase="r0", + task_names=selected, + qualification_family="gpt-5.6", + ) + + assert len(entries) == 1 + entry = entries[0] + assert entry["repetition"] == 0 + assert entry["expected_task_count"] == 10 + assert entry["task_names"] == selected + assert entry["phase"] == "r0" + assert entry["qualification_family"] == "gpt-5.6" + assert entry["leaderboard_eligible"] is False + assert entry["exclusion_reason"] == "r0_non_scoring_qualification" + assert entry["run_label"] == ( + "openclaw-gpt56-sol-high-smoke-10-r0-20260729" + ) + index = json.loads((tmp_path / "run-index.json").read_text()) + assert index["expected_task_count"] == 12 + assert index["planned_repetitions"] == [0] + + def test_plan_cli_accepts_repetition_count_and_filters() -> None: args = native_plan.parse_args( [ @@ -127,6 +166,37 @@ def test_plan_cli_accepts_repetition_count_and_filters() -> None: assert args.model_slugs == ["gpt56-sol"] +def test_plan_cli_accepts_r0_qualification_inputs() -> None: + task_args = [item for index in range(10) for item in ("--task", f"task-{index}")] + args = native_plan.parse_args( + [ + "--tasks-root", + "tasks", + "--output", + "run-index.json", + "--public-tasks-commit", + "abc", + "--run-date", + "20260729", + "--reasoning-effort", + "high", + "--phase", + "r0", + "--qualification-family", + "gpt-5.6", + "--harness", + "openclaw", + "--model", + "gpt56-sol", + *task_args, + ] + ) + + assert args.phase == "r0" + assert args.qualification_family == "gpt-5.6" + assert args.task_names == [f"task-{index}" for index in range(10)] + + def test_run_job_cli_accepts_repetition_six() -> None: args = parse_run_job_args( [ @@ -156,6 +226,35 @@ def test_run_job_cli_accepts_repetition_six() -> None: assert args.repetition == 6 +def test_run_job_cli_accepts_r0() -> None: + args = parse_run_job_args( + [ + "--tasks-root", + "tasks", + "--jobs-dir", + "jobs", + "--run-label", + "openclaw-gpt56-sol-high-smoke-10-r0-20260729", + "--harness", + "openclaw", + "--model-slug", + "gpt56-sol", + "--repetition", + "0", + "--expected-task-count", + "10", + "--public-tasks-commit", + "abc", + "--task-suite-path", + "combined tasks/tasks", + "--run-date", + "20260729", + ] + ) + + assert args.repetition == 0 + + def test_task_loader_accepts_rich_manifest_and_compose(tmp_path: Path) -> None: task_dir = tmp_path / "browser-task" (task_dir / "environment").mkdir(parents=True) @@ -301,6 +400,10 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["provider_model_id"] == "gpt-5.5" assert manifest["reasoning_effort"] == "high" assert manifest["judge_reasoning_effort"] == "high" + assert manifest["phase"] == "full" + assert manifest["qualification_family"] is None + assert manifest["leaderboard_eligible"] is None + assert manifest["exclusion_reason"] is None assert manifest["repair_mode"] is False assert manifest["rerun_of_canonical_run"] is None assert manifest["repair_task_names"] == [] @@ -309,6 +412,46 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["legacy_parity_validated_claim"] is False +def test_run_manifest_excludes_r0_from_leaderboard( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_RUN_PHASE", "r0") + monkeypatch.setenv("SHELLBENCH_QUALIFICATION_FAMILY", "gpt-5.6") + monkeypatch.setenv("SHELLBENCH_LEADERBOARD_ELIGIBLE", "false") + monkeypatch.setenv( + "SHELLBENCH_EXCLUSION_REASON", + "r0_non_scoring_qualification", + ) + run = RunSpec( + run_label="openclaw-gpt56-sol-high-smoke-10-r0-20260729", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=0, + expected_task_count=10, + run_date="20260729", + ) + + manifest = _run_manifest( + run, + public_tasks_commit="tasks-commit", + task_suite_path="combined tasks/tasks", + concurrency=2, + started_at="2026-07-29T00:00:00Z", + tasks_root=tmp_path, + tasks=[], + ) + + assert manifest["phase"] == "r0" + assert manifest["qualification_family"] == "gpt-5.6" + assert manifest["leaderboard_eligible"] is False + assert manifest["exclusion_reason"] == "r0_non_scoring_qualification" + + def test_run_manifest_scopes_parity_to_matching_route( tmp_path: Path, monkeypatch,