diff --git a/.agents/skills/shellbench-research-runbook/references/runbook.md b/.agents/skills/shellbench-research-runbook/references/runbook.md index dd8a1f0..17cd5dc 100644 --- a/.agents/skills/shellbench-research-runbook/references/runbook.md +++ b/.agents/skills/shellbench-research-runbook/references/runbook.md @@ -85,6 +85,22 @@ Before every campaign: Do not update a pin during a campaign. Start a new campaign ID when a harness, runner, task, provider route, or judge version changes. +For a trusted, unpublished OpenClaw build, pass its npm package tarball to the +fleet controller: + +```sh +python -m scripts.native_eval.fleet \ + ... \ + --openclaw-package-tarball /path/to/openclaw.tgz +``` + +Candidate campaigns must contain only OpenClaw runs. The controller validates +the npm package name and version, stages it as +`manifests/openclaw-candidate.tgz`, and pins its SHA-256 in campaign, toolchain, +and run provenance. Resume with the same tarball identity; a missing or changed +candidate is rejected before leasing. Registry installation remains the +default when the option is absent. + ## Campaign Identity Use a stable campaign ID: diff --git a/scripts/native_eval/audit.py b/scripts/native_eval/audit.py index 60b3077..5a2a5b9 100644 --- a/scripts/native_eval/audit.py +++ b/scripts/native_eval/audit.py @@ -23,6 +23,17 @@ def _read_object(path: Path) -> dict[str, Any]: return value +def _candidate_status( + expected: dict[str, Any] | None, + observed: Any, +) -> str: + if expected is None: + return "not_applicable" + if not isinstance(observed, dict): + return "missing" + return "match" if observed == expected else "mismatch" + + def build_metadata_supplements( run_index_path: Path, extracted_root: Path, @@ -36,8 +47,12 @@ def build_metadata_supplements( missing = [field for field, value in provenance.items() if value in (None, "")] if missing: raise ValueError(f"fleet metadata is missing: {', '.join(missing)}") + expected_candidate = fleet.get("openclaw_package") + if expected_candidate is not None and not isinstance(expected_candidate, dict): + raise ValueError("fleet openclaw_package must be an object") supplements = [] + candidate_counts = {"match": 0, "mismatch": 0, "missing": 0} for manifest_path in sorted(extracted_root.glob("*/shellbench_meta-*/run_manifest.json")): manifest = _read_object(manifest_path) run_label = str(manifest.get("run_label") or "") @@ -54,22 +69,31 @@ def build_metadata_supplements( f"{run_label} has conflicting {field}: " f"archive={archived!r}, fleet={expected!r}" ) - if additions: - supplements.append( - { - "run_label": run_label, - "archived_manifest": str(manifest_path.relative_to(extracted_root)), - "supplements": additions, - } - ) + candidate_status = _candidate_status( + expected_candidate, + manifest.get("openclaw_package"), + ) + if candidate_status != "not_applicable": + candidate_counts[candidate_status] += 1 + if additions or candidate_status != "not_applicable": + supplement = { + "run_label": run_label, + "archived_manifest": str(manifest_path.relative_to(extracted_root)), + "supplements": additions, + } + if candidate_status != "not_applicable": + supplement["openclaw_candidate_status"] = candidate_status + supplements.append(supplement) return { - "schema_version": 1, + "schema_version": 2, "generated_at_utc": datetime.now(UTC).isoformat(), "source_run_index": str(run_index_path), "extracted_root": str(extracted_root), "raw_archives_mutated": False, "provenance": provenance, + "openclaw_package": expected_candidate, + "openclaw_candidate_counts": candidate_counts, "runs": supplements, } diff --git a/scripts/native_eval/bootstrap_beast.sh b/scripts/native_eval/bootstrap_beast.sh index 8309721..3ee4d41 100755 --- a/scripts/native_eval/bootstrap_beast.sh +++ b/scripts/native_eval/bootstrap_beast.sh @@ -8,9 +8,23 @@ CODEX_VERSION="${CODEX_VERSION:-0.145.0}" CLAUDE_CODE_VERSION="${CLAUDE_CODE_VERSION:-2.1.220}" HERMES_COMMIT="${HERMES_COMMIT:-cb06017b1d6e1b9ae0cb35f99a48ffa6bcbaa828}" LITELLM_VERSION="${LITELLM_VERSION:-1.93.0}" +OPENCLAW_PACKAGE_TARBALL="${OPENCLAW_PACKAGE_TARBALL:-}" +OPENCLAW_PACKAGE_SHA256="${OPENCLAW_PACKAGE_SHA256:-}" +OPENCLAW_PACKAGE_VERSION="${OPENCLAW_PACKAGE_VERSION:-}" if [[ "$(id -u)" -ne 0 ]]; then - exec sudo -E bash "$0" "$@" + exec sudo -E env \ + TOOLCHAIN_ROOT="$TOOLCHAIN_ROOT" \ + NODE_VERSION="$NODE_VERSION" \ + OPENCLAW_VERSION="$OPENCLAW_VERSION" \ + CODEX_VERSION="$CODEX_VERSION" \ + CLAUDE_CODE_VERSION="$CLAUDE_CODE_VERSION" \ + HERMES_COMMIT="$HERMES_COMMIT" \ + LITELLM_VERSION="$LITELLM_VERSION" \ + OPENCLAW_PACKAGE_TARBALL="$OPENCLAW_PACKAGE_TARBALL" \ + OPENCLAW_PACKAGE_SHA256="$OPENCLAW_PACKAGE_SHA256" \ + OPENCLAW_PACKAGE_VERSION="$OPENCLAW_PACKAGE_VERSION" \ + bash "$0" "$@" fi export DEBIAN_FRONTEND=noninteractive @@ -67,6 +81,7 @@ EOF install_node_tools() { local node_root="$TOOLCHAIN_ROOT/node" + local openclaw_spec="openclaw@$OPENCLAW_VERSION" if [[ ! -x "$node_root/bin/node" ]] || \ [[ "$("$node_root/bin/node" --version)" != "v$NODE_VERSION" ]]; then rm -rf "$node_root" @@ -79,9 +94,19 @@ install_node_tools() { fi export PATH="$node_root/bin:$PATH" + if [[ -n "$OPENCLAW_PACKAGE_TARBALL" ]]; then + [[ -n "$OPENCLAW_PACKAGE_SHA256" && -n "$OPENCLAW_PACKAGE_VERSION" ]] + [[ -f "$OPENCLAW_PACKAGE_TARBALL" ]] + printf '%s %s\n' \ + "$OPENCLAW_PACKAGE_SHA256" "$OPENCLAW_PACKAGE_TARBALL" \ + | sha256sum -c - + [[ "$(tar -xOf "$OPENCLAW_PACKAGE_TARBALL" package/package.json | jq -r '.name')" == "openclaw" ]] + [[ "$(tar -xOf "$OPENCLAW_PACKAGE_TARBALL" package/package.json | jq -r '.version')" == "$OPENCLAW_PACKAGE_VERSION" ]] + openclaw_spec="$OPENCLAW_PACKAGE_TARBALL" + fi rm -rf "$TOOLCHAIN_ROOT/npm-packages" npm install --prefix "$TOOLCHAIN_ROOT/npm-packages" \ - "openclaw@$OPENCLAW_VERSION" \ + "$openclaw_spec" \ "@openai/codex@$CODEX_VERSION" \ "@anthropic-ai/claude-code@$CLAUDE_CODE_VERSION" } @@ -125,6 +150,16 @@ install_litellm() { } write_manifest() { + local openclaw_source_kind="registry" + local openclaw_package_version="$OPENCLAW_VERSION" + local openclaw_package_sha256="" + local openclaw_artifact_filename="" + if [[ -n "$OPENCLAW_PACKAGE_TARBALL" ]]; then + openclaw_source_kind="npm_tarball" + openclaw_package_version="$OPENCLAW_PACKAGE_VERSION" + openclaw_package_sha256="$OPENCLAW_PACKAGE_SHA256" + openclaw_artifact_filename="$(basename "$OPENCLAW_PACKAGE_TARBALL")" + fi export PATH="$TOOLCHAIN_ROOT/node/bin:$TOOLCHAIN_ROOT/npm-packages/node_modules/.bin:$TOOLCHAIN_ROOT/home/.local/bin:$PATH" jq -n \ --arg created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ @@ -135,10 +170,21 @@ write_manifest() { --arg hermes "$(hermes version | head -1)" \ --arg litellm "$("$TOOLCHAIN_ROOT/litellm-venv/bin/python" -c 'from importlib.metadata import version; print(version("litellm"))')" \ --arg hermes_commit "$HERMES_COMMIT" \ + --arg openclaw_source_kind "$openclaw_source_kind" \ + --arg openclaw_package_version "$openclaw_package_version" \ + --arg openclaw_package_sha256 "$openclaw_package_sha256" \ + --arg openclaw_artifact_filename "$openclaw_artifact_filename" \ '{ created_at_utc: $created_at, node: $node, openclaw: $openclaw, + openclaw_package: { + source_kind: $openclaw_source_kind, + package_name: "openclaw", + package_version: $openclaw_package_version, + sha256: (if $openclaw_package_sha256 == "" then null else $openclaw_package_sha256 end), + artifact_filename: (if $openclaw_artifact_filename == "" then null else $openclaw_artifact_filename end) + }, codex: $codex, claude_code: $claude_code, hermes: $hermes, diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 433d775..28624c4 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -5,7 +5,10 @@ import fcntl import hashlib import json +import os +import shutil import shlex +import stat import subprocess import sys import tarfile @@ -55,6 +58,7 @@ CRABBOX_READY_ATTEMPTS = 30 CRABBOX_READY_BACKOFF_SECONDS = 10 CRABBOX_INSPECT_ATTEMPTS = 4 +OPENCLAW_CANDIDATE_FILENAME = "openclaw-candidate.tgz" class CommandExecutor(Protocol): @@ -202,6 +206,7 @@ class FleetConfig: runner_root: Path task_archive: Path env_file: Path + openclaw_package_tarball: Path | None = None max_leases: int = 10 max_attempts: int = 2 task_concurrency: int = 16 @@ -263,6 +268,21 @@ def update_root(self, **changes: Any) -> None: self.data.update(changes) self.save() + def update_fleet_and_runs( + self, + *, + fleet: dict[str, Any], + run_changes: dict[str, Any] | None = None, + ) -> None: + with self._thread_lock: + self.data["fleet"] = fleet + if run_changes: + updated_at = utc_now() + for entry in self.data["runs"]: + entry.update(run_changes) + entry["updated_at_utc"] = updated_at + self.save() + def get(self, run_label: str) -> dict[str, Any]: with self._thread_lock: for entry in self.data["runs"]: @@ -326,6 +346,8 @@ def __init__( self.runner_commit = "" self.runner_archive_sha256 = "" self.task_archive_sha256 = "" + self.openclaw_package: dict[str, str] | None = None + self.openclaw_package_tarball: Path | None = None self.crabbox_cli_version = "" self._dispatch_lock = threading.Lock() self._cleanup_lock = threading.Lock() @@ -384,8 +406,10 @@ def _prepare_local_layout(self) -> None: def _validate_plan(self) -> None: suite_expected = int(self._store.data["expected_task_count"]) + harnesses: set[str] = set() for entry in self._store.all_entries(): run = self._run_spec(entry) + harnesses.add(run.harness) task_names = entry.get("task_names") phase = str(entry.get("phase") or "full") if task_names is None: @@ -441,6 +465,10 @@ def _validate_plan(self) -> None: f"{run.run_label} must set judge_reasoning_effort to " "low, medium, high, or xhigh" ) + if self.config.openclaw_package_tarball and harnesses != {"openclaw"}: + raise FleetError( + "--openclaw-package-tarball requires an OpenClaw-only run index" + ) def _prepare_inputs(self) -> None: for path, description in ( @@ -500,6 +528,7 @@ def _prepare_inputs(self) -> None: raise FleetError(f"invalid archive: {archive}") from exc self.runner_archive_sha256 = _sha256(self.runner_archive) self.task_archive_sha256 = _sha256(self.config.task_archive) + self._prepare_openclaw_package() version = self._checked( [self.config.crabbox_bin, "--version"], capture_output=True, @@ -507,6 +536,71 @@ def _prepare_inputs(self) -> None: ) self.crabbox_cli_version = version.stdout.strip() + def _prepare_openclaw_package(self) -> None: + configured = self.config.openclaw_package_tarball + candidate = ( + _inspect_openclaw_package_tarball(configured) + if configured is not None + else None + ) + if candidate is not None: + mismatched_runs = [ + str(entry["run_label"]) + for entry in self._store.all_entries() + if entry.get("harness_version") != candidate["package_version"] + ] + if mismatched_runs: + raise FleetError( + "OpenClaw candidate package version does not match planned " + f"harness_version for: {', '.join(mismatched_runs)}" + ) + fleet = self._store.data.get("fleet") + existing_fleet = fleet if isinstance(fleet, dict) else {} + existing = existing_fleet.get("openclaw_package") + run_identities = [ + entry.get("openclaw_package") for entry in self._store.all_entries() + ] + + if existing is not None: + if candidate is None: + raise FleetError( + "existing campaign requires --openclaw-package-tarball" + ) + if existing != candidate: + raise FleetError("OpenClaw candidate identity changed on resume") + if any( + identity is not None and identity != existing + for identity in run_identities + ): + raise FleetError( + "existing campaign run mismatches OpenClaw candidate identity" + ) + elif candidate is not None and existing_fleet: + raise FleetError( + "existing campaign is missing OpenClaw candidate identity" + ) + elif any(identity is not None for identity in run_identities): + raise FleetError( + "run index has OpenClaw candidate identity without campaign identity" + ) + + self.openclaw_package = candidate + if candidate is None: + return + + manifests = self.config.local_root / "manifests" + staged = manifests / OPENCLAW_CANDIDATE_FILENAME + source = configured.resolve() + if staged.exists() and _sha256(staged) != candidate["sha256"]: + raise FleetError(f"staged OpenClaw candidate has changed: {staged}") + if not staged.exists(): + shutil.copyfile(source, staged) + staged.chmod(0o600) + if _sha256(staged) != candidate["sha256"]: + raise FleetError("staged OpenClaw candidate failed SHA-256 verification") + atomic_write_json(manifests / "openclaw_package.json", candidate) + self.openclaw_package_tarball = staged + def _record_fleet_metadata(self) -> None: assert self.runner_archive is not None metadata = dict(self._store.data.get("fleet") or {}) @@ -536,7 +630,17 @@ def _record_fleet_metadata(self) -> None: "controller_started_at_utc": utc_now(), } ) - self._store.update_root(fleet=metadata) + if self.openclaw_package is not None: + metadata["openclaw_package"] = self.openclaw_package + run_changes = ( + {"openclaw_package": self.openclaw_package} + if self.openclaw_package is not None + else None + ) + self._store.update_fleet_and_runs( + fleet=metadata, + run_changes=run_changes, + ) def _execute_entry(self, run_label: str) -> bool: entry = self._store.get(run_label) @@ -828,10 +932,20 @@ def _hydrate_lease(self, lease: Lease) -> None: remote_runner_archive = f"/tmp/{lease.slug}-runner.tar.gz" remote_task_archive = f"/tmp/{lease.slug}-tasks.tar.gz" remote_env = f"/tmp/{lease.slug}-provider.env" + remote_openclaw_package = ( + f"{self.config.remote_root}/artifacts/{OPENCLAW_CANDIDATE_FILENAME}" + if self.openclaw_package_tarball is not None + else "" + ) self._checked( self._ssh_command( lease, - ["mkdir", "-p", self.config.remote_root], + [ + "mkdir", + "-p", + self.config.remote_root, + f"{self.config.remote_root}/artifacts", + ], ), capture_output=True, description=f"prepare {lease.slug}", @@ -839,7 +953,11 @@ def _hydrate_lease(self, lease: Lease) -> None: for local_path, remote_path in ( (self.runner_archive, remote_runner_archive), (self.config.task_archive, remote_task_archive), - (self.config.env_file, remote_env), + *( + ((self.openclaw_package_tarball, remote_openclaw_package),) + if self.openclaw_package_tarball is not None + else () + ), ): self._checked( self._scp_command(lease, local_path, remote_path), @@ -852,24 +970,45 @@ def _hydrate_lease(self, lease: Lease) -> None: root=$1 runner_archive=$2 task_archive=$3 -source_env=$4 -runner_commit=$5 -runner_sha256=$6 -task_sha256=$7 +runner_commit=$4 +runner_sha256=$5 +task_sha256=$6 +candidate_archive=$7 +candidate_sha256=$8 +candidate_version=$9 +shift 9 printf '%s %s\n' "$runner_sha256" "$runner_archive" | sha256sum -c - printf '%s %s\n' "$task_sha256" "$task_archive" | sha256sum -c - +if [ -n "$candidate_archive" ]; then + printf '%s %s\n' "$candidate_sha256" "$candidate_archive" | sha256sum -c - +fi rm -rf "$root/runner.new" "$root/public-tasks.new" mkdir -p "$root/runner.new" "$root/public-tasks.new" "$root/secrets" tar -xzf "$runner_archive" -C "$root/runner.new" tar -xzf "$task_archive" -C "$root/public-tasks.new" -install -m 0600 "$source_env" "$root/secrets/provider.env" rm -rf "$root/runner" "$root/public-tasks" mv "$root/runner.new" "$root/runner" mv "$root/public-tasks.new" "$root/public-tasks" printf '%s\n' "$runner_commit" > "$root/runner.commit" -rm -f "$runner_archive" "$task_archive" "$source_env" -bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" +rm -f "$runner_archive" "$task_archive" +if [ -n "$candidate_archive" ]; then + env \ + OPENCLAW_PACKAGE_TARBALL="$candidate_archive" \ + OPENCLAW_PACKAGE_SHA256="$candidate_sha256" \ + OPENCLAW_PACKAGE_VERSION="$candidate_version" \ + bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" +else + bash "$root/runner/scripts/native_eval/bootstrap_beast.sh" +fi """ + candidate_sha256 = ( + self.openclaw_package["sha256"] if self.openclaw_package else "" + ) + candidate_version = ( + self.openclaw_package["package_version"] + if self.openclaw_package + else "" + ) self._checked( self._ssh_command( lease, @@ -881,15 +1020,41 @@ def _hydrate_lease(self, lease: Lease) -> None: self.config.remote_root, remote_runner_archive, remote_task_archive, - remote_env, self.runner_commit, self.runner_archive_sha256, self.task_archive_sha256, + remote_openclaw_package, + candidate_sha256, + candidate_version, ], ), capture_output=False, description=f"bootstrap {lease.slug}", ) + self._checked( + self._scp_command(lease, self.config.env_file, remote_env), + capture_output=True, + description=f"sync provider environment to {lease.slug}", + ) + self._checked( + self._ssh_command( + lease, + [ + "bash", + "-c", + """ +set -euo pipefail +install -m 0600 "$1" "$2/secrets/provider.env" +rm -f "$1" +""", + "fleet-provider-env", + remote_env, + self.config.remote_root, + ], + ), + capture_output=True, + description=f"install provider environment on {lease.slug}", + ) self._store.update( self._run_label_for_lease(lease.lease_id), status="ready", @@ -1474,6 +1639,8 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "artifacts": [], "created_at_utc": utc_now(), } + if self.openclaw_package is not None: + rerun["openclaw_package"] = self.openclaw_package for metadata_field in ( "task_names", "rerun_of_canonical_run", @@ -1585,6 +1752,44 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _inspect_openclaw_package_tarball(path: Path) -> dict[str, str]: + try: + mode = path.stat().st_mode + except OSError as exc: + raise FleetError(f"OpenClaw package tarball is unavailable: {path}") from exc + if not stat.S_ISREG(mode) or not os.access(path, os.R_OK): + raise FleetError( + f"OpenClaw package tarball must be a readable regular file: {path}" + ) + try: + with tarfile.open(path, "r:*") as handle: + member = handle.getmember("package/package.json") + if not member.isfile() or member.size > 1024 * 1024: + raise FleetError( + "OpenClaw package tarball has an invalid package/package.json" + ) + package_file = handle.extractfile(member) + if package_file is None: + raise FleetError( + "OpenClaw package tarball is missing package/package.json" + ) + package = json.loads(package_file.read()) + except (KeyError, OSError, tarfile.TarError, json.JSONDecodeError) as exc: + raise FleetError(f"invalid OpenClaw npm package tarball: {path}") from exc + if not isinstance(package, dict) or package.get("name") != "openclaw": + raise FleetError("OpenClaw package tarball must have name=openclaw") + version = package.get("version") + if not isinstance(version, str) or not version.strip(): + raise FleetError("OpenClaw package tarball must declare a package version") + return { + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": version, + "sha256": _sha256(path), + "artifact_filename": OPENCLAW_CANDIDATE_FILENAME, + } + + def _validate_model_values(values: dict[str, int], name: str) -> None: for model_slug, value in values.items(): if not model_slug: @@ -1640,6 +1845,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--runner-root", type=Path, required=True) parser.add_argument("--task-archive", type=Path, required=True) parser.add_argument("--env-file", type=Path, required=True) + parser.add_argument("--openclaw-package-tarball", type=Path) parser.add_argument("--max-leases", type=int, default=10) parser.add_argument("--max-attempts", type=int, default=2) parser.add_argument("--task-concurrency", type=int, default=16) @@ -1714,6 +1920,7 @@ def main(argv: list[str] | None = None) -> int: runner_root=args.runner_root, task_archive=args.task_archive, env_file=args.env_file, + openclaw_package_tarball=args.openclaw_package_tarball, max_leases=args.max_leases, max_attempts=args.max_attempts, task_concurrency=args.task_concurrency, diff --git a/scripts/native_eval/research_audit.py b/scripts/native_eval/research_audit.py index 381df62..8442fbd 100644 --- a/scripts/native_eval/research_audit.py +++ b/scripts/native_eval/research_audit.py @@ -17,6 +17,11 @@ "harness_version", "installed_harness_version", "harness_version_status", + "openclaw_candidate_status", + "expected_openclaw_package_version", + "expected_openclaw_package_sha256", + "installed_openclaw_package_version", + "installed_openclaw_package_sha256", "model_slug", "expected_model_id", "reasoning_effort", @@ -91,6 +96,11 @@ "harness_version", "installed_harness_version", "harness_version_status", + "openclaw_candidate_status", + "expected_openclaw_package_version", + "expected_openclaw_package_sha256", + "installed_openclaw_package_version", + "installed_openclaw_package_sha256", "model_slug", "expected_model_id", "reasoning_effort", @@ -362,6 +372,22 @@ def _version_status(requested: str, installed: str) -> str: return "mismatch" +def _candidate_status( + *, + harness: str, + expected: Any, + installed: Any, + run_manifest: Any, +) -> str: + if harness != "openclaw" or not isinstance(expected, dict): + return "not_applicable" + if not isinstance(installed, dict) or not isinstance(run_manifest, dict): + return "missing" + if installed == expected and run_manifest == expected: + return "match" + return "mismatch" + + def export_research_tables( *, run_index_path: Path, @@ -400,6 +426,44 @@ def export_research_tables( requested_harness_version, installed_harness_version, ) + expected_openclaw_package = entry.get("openclaw_package") + installed_openclaw_package = toolchain.get("openclaw_package") + job_manifest = ( + _read_json(job_dir / "run_manifest.json") + if job_dir is not None + else None + ) + run_openclaw_package = ( + job_manifest.get("openclaw_package") + if isinstance(job_manifest, dict) + else None + ) + openclaw_candidate_status = _candidate_status( + harness=harness, + expected=expected_openclaw_package, + installed=installed_openclaw_package, + run_manifest=run_openclaw_package, + ) + expected_candidate_version = ( + expected_openclaw_package.get("package_version") + if isinstance(expected_openclaw_package, dict) + else None + ) + expected_candidate_sha256 = ( + expected_openclaw_package.get("sha256") + if isinstance(expected_openclaw_package, dict) + else None + ) + installed_candidate_version = ( + installed_openclaw_package.get("package_version") + if isinstance(installed_openclaw_package, dict) + else None + ) + installed_candidate_sha256 = ( + installed_openclaw_package.get("sha256") + if isinstance(installed_openclaw_package, dict) + else None + ) proxy_log_path = proxy_logs.get(run_label) counters: Counter[str] = Counter() if job_dir is not None: @@ -512,6 +576,11 @@ def export_research_tables( "harness_version": requested_harness_version, "installed_harness_version": installed_harness_version, "harness_version_status": harness_version_status, + "openclaw_candidate_status": openclaw_candidate_status, + "expected_openclaw_package_version": expected_candidate_version, + "expected_openclaw_package_sha256": expected_candidate_sha256, + "installed_openclaw_package_version": installed_candidate_version, + "installed_openclaw_package_sha256": installed_candidate_sha256, "model_slug": entry.get("model_slug"), "expected_model_id": expected_model_id, "reasoning_effort": entry.get("reasoning_effort"), @@ -549,6 +618,7 @@ def export_research_tables( passed = result_count > 0 and ( counters["match"] == result_count and counters["real_trace"] == result_count + and openclaw_candidate_status in {"match", "not_applicable"} ) run_rows.append( { @@ -557,6 +627,11 @@ def export_research_tables( "harness_version": requested_harness_version, "installed_harness_version": installed_harness_version, "harness_version_status": harness_version_status, + "openclaw_candidate_status": openclaw_candidate_status, + "expected_openclaw_package_version": expected_candidate_version, + "expected_openclaw_package_sha256": expected_candidate_sha256, + "installed_openclaw_package_version": installed_candidate_version, + "installed_openclaw_package_sha256": installed_candidate_sha256, "model_slug": entry.get("model_slug"), "expected_model_id": entry.get("model_id"), "reasoning_effort": entry.get("reasoning_effort"), @@ -601,6 +676,15 @@ def export_research_tables( "identity_audit_fail_count": sum( row["model_identity_audit_passed"] is not True for row in run_rows ), + "openclaw_candidate_match_count": sum( + row["openclaw_candidate_status"] == "match" for row in run_rows + ), + "openclaw_candidate_mismatch_count": sum( + row["openclaw_candidate_status"] == "mismatch" for row in run_rows + ), + "openclaw_candidate_missing_count": sum( + row["openclaw_candidate_status"] == "missing" for row in run_rows + ), "r0_run_count": sum(row["phase"] == "r0" for row in run_rows), "scoring_run_count": sum( row["leaderboard_eligible"] is not False for row in run_rows diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..b548edf 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -48,6 +48,7 @@ async def run_job( raise FileNotFoundError(f"Toolchain not found: {toolchain_root}") if not proxy_key: raise ValueError("SHELLBENCH_PROXY_KEY is required") + openclaw_package = _toolchain_openclaw_package(toolchain_root) job_dir = jobs_dir / run.run_label job_dir.mkdir(parents=True, exist_ok=False) @@ -62,6 +63,7 @@ async def run_job( tasks_root=tasks_root, tasks=tasks, rerun_of_canonical_run=rerun_of_canonical_run, + openclaw_package=openclaw_package, ) atomic_write_json(job_dir / "run_manifest.json", manifest) atomic_write_json( @@ -214,12 +216,14 @@ def _run_manifest( tasks_root: Path, tasks: list[TaskSpec], rerun_of_canonical_run: str | None = None, + openclaw_package: dict[str, Any] | None = None, ) -> dict[str, Any]: parity_validation, parity_validated = _parity_metadata(run) return { "run_label": run.run_label, "harness": run.harness, "harness_version": run.harness_version, + "openclaw_package": openclaw_package if run.harness == "openclaw" else None, "model_slug": run.model_slug, "model_id": run.model_id, "provider_model_id": run.model_id, @@ -296,6 +300,16 @@ def _run_manifest( } +def _toolchain_openclaw_package(toolchain_root: Path) -> dict[str, Any] | None: + manifest_path = toolchain_root / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return None + package = manifest.get("openclaw_package") if isinstance(manifest, dict) else None + return package if isinstance(package, dict) else None + + def _parity_metadata(run: RunSpec) -> tuple[dict[str, Any] | None, bool]: raw = os.environ.get("SHELLBENCH_PARITY_VALIDATION_JSON", "").strip() if not raw: diff --git a/tests/test_native_eval_audit.py b/tests/test_native_eval_audit.py index 2f4160e..db76821 100644 --- a/tests/test_native_eval_audit.py +++ b/tests/test_native_eval_audit.py @@ -82,3 +82,68 @@ def test_build_metadata_supplements_rejects_conflicting_provenance(tmp_path: Pat with pytest.raises(ValueError, match="conflicting execution_mode"): build_metadata_supplements(run_index, extracted) + + +@pytest.mark.parametrize( + ("observed", "status"), + [ + (None, "missing"), + ( + { + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": "different", + "sha256": "different", + "artifact_filename": "openclaw-candidate.tgz", + }, + "mismatch", + ), + ("expected", "match"), + ], +) +def test_build_metadata_supplements_classifies_openclaw_candidate( + tmp_path: Path, + observed: dict | str | None, + status: str, +) -> None: + candidate = { + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": "2026.7.29-candidate.1", + "sha256": "candidate-sha", + "artifact_filename": "openclaw-candidate.tgz", + } + run_index = tmp_path / "run_index.json" + extracted = tmp_path / "extracted" + _write_json( + run_index, + { + "fleet": { + "execution_mode": "native", + "harbor_reference_commit": "harbor-sha", + "judge_model_id": "gpt-5.5", + "openclaw_package": candidate, + } + }, + ) + manifest = { + "run_label": "candidate-run", + "execution_mode": "native", + "harbor_reference_commit": "harbor-sha", + "judge_model_id": "gpt-5.5", + } + if observed is not None: + manifest["openclaw_package"] = ( + candidate if observed == "expected" else observed + ) + _write_json( + extracted / "run" / "shellbench_meta-candidate-run" / "run_manifest.json", + manifest, + ) + + report = build_metadata_supplements(run_index, extracted) + + assert report["runs"][0]["openclaw_candidate_status"] == status + assert report["openclaw_candidate_counts"][status] == 1 + if status == "missing": + assert "openclaw_package" not in report["runs"][0]["supplements"] diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index b9855ff..6d0ac42 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -1,6 +1,8 @@ from __future__ import annotations +import hashlib import json +import os import shlex import subprocess import tarfile @@ -29,10 +31,11 @@ def _run_spec( expected_task_count: int = 2, model_slug: str = "gpt55", repetition: int = 1, + harness: str = "openclaw", ) -> RunSpec: return RunSpec( run_label=label, - harness="openclaw", + harness=harness, harness_version="test", model_slug=model_slug, model_id=f"provider/{model_slug}", @@ -243,6 +246,31 @@ def _write_archive(path: Path) -> None: handle.add(source, arcname=".") +def _write_openclaw_package( + path: Path, + *, + name: str = "openclaw", + version: str = "test", + marker: str | None = None, +) -> dict[str, str]: + source = path.parent / f"{path.stem}-package" + package_root = source / "package" + package_root.mkdir(parents=True) + (package_root / "package.json").write_text( + json.dumps({"name": name, "version": version, "candidate_marker": marker}), + encoding="utf-8", + ) + with tarfile.open(path, "w:gz") as handle: + handle.add(package_root, arcname="package") + return { + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": version, + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "artifact_filename": "openclaw-candidate.tgz", + } + + def _write_final( local_root: Path, run_label: str, @@ -500,6 +528,7 @@ def _config( warmup_capacity_attempts: int = 12, warmup_capacity_backoff_seconds: float = 0, parity_validated_routes: frozenset[tuple[str, str]] = frozenset(), + openclaw_package_tarball: Path | None = None, ) -> FleetConfig: runner_archive = tmp_path / "runner.tar.gz" task_archive = tmp_path / "tasks.tar.gz" @@ -515,6 +544,7 @@ def _config( runner_commit="runner-commit", task_archive=task_archive, env_file=env_file, + openclaw_package_tarball=openclaw_package_tarball, max_leases=max_leases, max_attempts=max_attempts, task_concurrency=task_concurrency, @@ -528,6 +558,231 @@ def _config( ) +def test_candidate_package_is_validated_staged_and_bootstrapped( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [_planned(_run_spec(label))]) + package = tmp_path / "private-build-name.tgz" + identity = _write_openclaw_package(package) + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=package, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + staged = config.local_root / "manifests" / "openclaw-candidate.tgz" + assert staged.read_bytes() == package.read_bytes() + assert json.loads( + (config.local_root / "manifests" / "openclaw_package.json").read_text() + ) == identity + index = json.loads(run_index.read_text(encoding="utf-8")) + assert index["fleet"]["openclaw_package"] == identity + assert index["runs"][0]["openclaw_package"] == identity + assert str(package.resolve()) not in json.dumps(index) + scp_commands = [command for command in executor.commands if command[0] == "scp"] + candidate_scp = next( + command for command in scp_commands if "openclaw-candidate.tgz" in command[-1] + ) + assert candidate_scp[-2] == str(staged) + bootstrap = next( + command + for command in executor.commands + if command[0] == "ssh" and "OPENCLAW_PACKAGE_TARBALL" in command[-1] + ) + assert identity["sha256"] in bootstrap[-1] + assert identity["package_version"] in bootstrap[-1] + assert "TOPSECRET" not in bootstrap[-1] + + +def test_bootstrap_preserves_candidate_identity_across_sudo(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_id = fake_bin / "id" + fake_id.write_text("#!/bin/sh\nprintf '1000\\n'\n", encoding="utf-8") + fake_id.chmod(0o755) + fake_sudo = fake_bin / "sudo" + fake_sudo.write_text( + "#!/bin/sh\nprintf '%s\\n' \"$@\"\n", + encoding="utf-8", + ) + fake_sudo.chmod(0o755) + candidate = tmp_path / "candidate.tgz" + candidate.write_bytes(b"candidate") + + result = subprocess.run( + ["bash", "scripts/native_eval/bootstrap_beast.sh"], + cwd=Path(__file__).resolve().parents[1], + env={ + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "OPENCLAW_PACKAGE_TARBALL": str(candidate), + "OPENCLAW_PACKAGE_SHA256": "candidate-sha256", + "OPENCLAW_PACKAGE_VERSION": "2026.7.2", + }, + check=True, + capture_output=True, + text=True, + ) + + assert result.stdout.splitlines()[:5] == [ + "-E", + "env", + "TOOLCHAIN_ROOT=/opt/shellbench-native", + "NODE_VERSION=22.23.1", + "OPENCLAW_VERSION=2026.7.1-2", + ] + assert f"OPENCLAW_PACKAGE_TARBALL={candidate}" in result.stdout + assert "OPENCLAW_PACKAGE_SHA256=candidate-sha256" in result.stdout + assert "OPENCLAW_PACKAGE_VERSION=2026.7.2" in result.stdout + + +def test_candidate_package_rejects_wrong_npm_identity_before_leasing( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [_planned(_run_spec(label))]) + package = tmp_path / "wrong.tgz" + _write_openclaw_package(package, name="not-openclaw") + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=package, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + with pytest.raises(FleetError, match="name=openclaw"): + FleetController(config, executor=executor).run() + assert not any(command[:2] == ["crabbox", "warmup"] for command in executor.commands) + + +def test_candidate_package_rejects_non_openclaw_run_index_before_leasing( + tmp_path: Path, +) -> None: + label = "codex-gpt55-full-2-r1-20260727" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index( + run_index, + [_planned(_run_spec(label, harness="codex"))], + ) + package = tmp_path / "candidate.tgz" + _write_openclaw_package(package) + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=package, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + with pytest.raises(FleetError, match="OpenClaw-only"): + FleetController(config, executor=executor).run() + assert not any(command[:2] == ["crabbox", "warmup"] for command in executor.commands) + + +def test_candidate_package_rejects_planned_harness_version_mismatch( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run = _planned(_run_spec(label)) + run["harness_version"] = "2026.7.28" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + package = tmp_path / "candidate.tgz" + _write_openclaw_package(package, version="2026.7.29") + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=package, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + with pytest.raises(FleetError, match="does not match planned harness_version"): + FleetController(config, executor=executor).run() + assert not any(command[:2] == ["crabbox", "warmup"] for command in executor.commands) + + +def test_candidate_package_resume_rejects_changed_identity( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run = _planned(_run_spec(label)) + original = tmp_path / "original.tgz" + original_identity = _write_openclaw_package(original) + run["openclaw_package"] = original_identity + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + index = json.loads(run_index.read_text(encoding="utf-8")) + index["fleet"] = {"openclaw_package": original_identity} + run_index.write_text(json.dumps(index), encoding="utf-8") + changed = tmp_path / "changed.tgz" + _write_openclaw_package(changed, marker="changed") + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=changed, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + with pytest.raises(FleetError, match="identity changed"): + FleetController(config, executor=executor).run() + assert not any(command[:2] == ["crabbox", "warmup"] for command in executor.commands) + + +def test_candidate_package_resume_requires_original_identity_input( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run = _planned(_run_spec(label)) + package = tmp_path / "candidate.tgz" + identity = _write_openclaw_package(package) + run["openclaw_package"] = identity + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + index = json.loads(run_index.read_text(encoding="utf-8")) + index["fleet"] = {"openclaw_package": identity} + run_index.write_text(json.dumps(index), encoding="utf-8") + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + with pytest.raises( + FleetError, + match="requires --openclaw-package-tarball", + ): + FleetController(config, executor=executor).run() + assert not any(command[:2] == ["crabbox", "warmup"] for command in executor.commands) + + +def test_candidate_package_resume_repairs_missing_run_identity( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-full-2-r1-20260727" + run = _planned(_run_spec(label)) + package = tmp_path / "candidate.tgz" + identity = _write_openclaw_package(package) + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + index = json.loads(run_index.read_text(encoding="utf-8")) + index["fleet"] = {"openclaw_package": identity} + run_index.write_text(json.dumps(index), encoding="utf-8") + config = _config( + tmp_path, + run_index, + openclaw_package_tarball=package, + ) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + repaired = json.loads(run_index.read_text(encoding="utf-8")) + assert repaired["fleet"]["openclaw_package"] == identity + assert repaired["runs"][0]["openclaw_package"] == identity + + def test_controller_runs_bounded_wave_and_stops_after_verified_export( tmp_path: Path, ) -> None: @@ -1077,6 +1332,8 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: str(tmp_path / "tasks.tar.gz"), "--env-file", str(tmp_path / ".env"), + "--openclaw-package-tarball", + str(tmp_path / "openclaw.tgz"), "--model-max-runs", "fable5=1", "--model-max-runs", @@ -1105,6 +1362,7 @@ def test_parse_args_accepts_repeatable_model_limits(tmp_path: Path) -> None: assert args.parity_validated is True assert args.parity_validated_routes == frozenset({("codex", "gpt55")}) assert args.model_task_concurrency == {"fable5": 2} + assert args.openclaw_package_tarball == tmp_path / "openclaw.tgz" @pytest.mark.parametrize( diff --git a/tests/test_native_eval_research_audit.py b/tests/test_native_eval_research_audit.py index 8e37557..fdb3389 100644 --- a/tests/test_native_eval_research_audit.py +++ b/tests/test_native_eval_research_audit.py @@ -180,3 +180,81 @@ def test_research_audit_fails_identity_when_trace_is_missing(tmp_path: Path) -> assert summary["identity_audit_pass_count"] == 0 assert summary["identity_audit_fail_count"] == 1 + + +def test_research_audit_requires_matching_openclaw_candidate_provenance( + tmp_path: Path, +) -> None: + candidate = { + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": "2026.7.29-candidate.1", + "sha256": "candidate-sha", + "artifact_filename": "openclaw-candidate.tgz", + } + statuses = { + "match": candidate, + "mismatch": {**candidate, "sha256": "different"}, + "missing": None, + } + for status, installed in statuses.items(): + run_label = f"openclaw-candidate-{status}" + root = tmp_path / status + extracted = root / "extracted" + job_dir = extracted / run_label + trial_dir = job_dir / "task__abc" + run_entry = { + "run_label": run_label, + "harness": "openclaw", + "harness_version": "2026.7.29-candidate.1", + "model_slug": "gpt56-sol", + "model_id": "gpt-5.6-sol", + "repetition": 1, + "expected_task_count": 1, + "openclaw_package": candidate, + } + _write_json(root / "run-index.json", {"runs": [run_entry]}) + run_manifest = {"run_label": run_label} + toolchain = {"openclaw": "openclaw 2026.7.29-candidate.1"} + if installed is not None: + run_manifest["openclaw_package"] = installed + toolchain["openclaw_package"] = installed + _write_json(job_dir / "run_manifest.json", run_manifest) + _write_json( + extracted / f"shellbench_meta-{run_label}" / "toolchain_manifest.json", + toolchain, + ) + _write_json( + trial_dir / "result.json", + { + "task_id": {"path": "/tasks/example-task"}, + "agent_result": { + "trajectory_status": "real", + "runtime_model_name": "gpt-5.6-sol", + "canonical_model_identity": True, + }, + }, + ) + _write_json( + trial_dir / "agent" / "trajectory.json", + { + "agent": {"model_name": "openai/gpt-5.6-sol"}, + "steps": [], + "extra": {"observed_models": ["gpt-5.6-sol"]}, + }, + ) + + summary = export_research_tables( + run_index_path=root / "run-index.json", + extracted_root=extracted, + output_dir=root / "analysis", + ) + with (root / "analysis" / "model_identity_audit.csv").open( + newline="", + encoding="utf-8", + ) as handle: + row = next(csv.DictReader(handle)) + + assert row["openclaw_candidate_status"] == status + assert row["model_identity_audit_passed"] == str(status == "match") + assert summary[f"openclaw_candidate_{status}_count"] == 1 diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..fe2f46d 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -395,6 +395,13 @@ def test_run_manifest_records_native_audit_metadata( started_at="2026-07-27T00:00:00Z", tasks_root=tmp_path, tasks=[], + openclaw_package={ + "source_kind": "npm_tarball", + "package_name": "openclaw", + "package_version": "2026.7.29-candidate.1", + "sha256": "candidate-sha", + "artifact_filename": "openclaw-candidate.tgz", + }, ) assert manifest["execution_mode"] == "native" @@ -415,6 +422,7 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["parity_validated"] is False assert manifest["parity_validation"] is None assert manifest["legacy_parity_validated_claim"] is False + assert manifest["openclaw_package"]["sha256"] == "candidate-sha" def test_run_manifest_excludes_r0_from_leaderboard(