From 7f31256da91aaa44d0b735ee42f1f9a45fc10dcc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 29 Jul 2026 22:31:28 +0200 Subject: [PATCH 1/5] fix(native-eval): make tool search controls explicit --- scripts/native_eval/fleet.py | 32 +++++++++++++- scripts/native_eval/harnesses.py | 12 +++++- scripts/native_eval/models.py | 1 + scripts/native_eval/run_job.py | 9 ++++ tests/test_native_eval_fleet.py | 42 +++++++++++++++++- tests/test_native_eval_runner.py | 73 ++++++++++++++++++++++++++++++++ 6 files changed, 165 insertions(+), 4 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index 433d775..feb70b2 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -48,6 +48,7 @@ ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} +OPENCLAW_TOOL_SEARCH_MODES = {"directory", "code"} # Crabbox's coordinator release path retries five 60-second requests with # bounded backoff. Give it enough time to finish instead of leaking live AWS # leases after a verified export. @@ -441,6 +442,18 @@ def _validate_plan(self) -> None: f"{run.run_label} must set judge_reasoning_effort to " "low, medium, high, or xhigh" ) + tool_search_mode = str(entry.get("openclaw_tool_search_mode") or "") + if tool_search_mode: + if run.harness != "openclaw": + raise FleetError( + f"{run.run_label} openclaw_tool_search_mode requires the " + "OpenClaw harness" + ) + if tool_search_mode not in OPENCLAW_TOOL_SEARCH_MODES: + raise FleetError( + f"{run.run_label} must set openclaw_tool_search_mode to " + f"one of {sorted(OPENCLAW_TOOL_SEARCH_MODES)}" + ) def _prepare_inputs(self) -> None: for path, description in ( @@ -958,7 +971,8 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: shift 9 parity_validated=$1 parity_validation_json=$2 -shift 2 +openclaw_tool_search_mode=$3 +shift 3 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" stderr="$root/run-logs/$label.stderr.log" @@ -981,6 +995,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_EXCLUSION_REASON=$exclusion_reason" \ "SHELLBENCH_PARITY_VALIDATED=$parity_validated" \ "SHELLBENCH_PARITY_VALIDATION_JSON=$parity_validation_json" \ + "SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE=$openclaw_tool_search_mode" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: separators=(",", ":"), sort_keys=True, ) + openclaw_tool_search_mode = str( + entry.get("openclaw_tool_search_mode") or "" + ) command = self._ssh_command( lease, [ @@ -1058,6 +1076,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: exclusion_reason, str(self.config.parity_validated).lower(), parity_validation, + openclaw_tool_search_mode, *args, ], ) @@ -1463,6 +1482,9 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "run_label": label, "reasoning_effort": entry.get("reasoning_effort"), "judge_reasoning_effort": entry.get("judge_reasoning_effort"), + "openclaw_tool_search_mode": entry.get( + "openclaw_tool_search_mode" + ), "phase": entry.get("phase"), "qualification_family": entry.get("qualification_family"), "attempt": next_attempt, @@ -1478,6 +1500,7 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "task_names", "rerun_of_canonical_run", "repair_classifications", + "openclaw_tool_search_mode", ): if metadata_field in entry: rerun[metadata_field] = copy.deepcopy(entry[metadata_field]) @@ -1498,7 +1521,12 @@ def _matrix_satisfied(self) -> bool: def _run_spec(self, entry: dict[str, Any]) -> RunSpec: try: - return RunSpec(**{field: entry[field] for field in RUN_SPEC_FIELDS}) + return RunSpec( + **{field: entry[field] for field in RUN_SPEC_FIELDS}, + openclaw_tool_search_mode=entry.get( + "openclaw_tool_search_mode" + ), + ) except KeyError as exc: raise FleetError( f"run entry {entry.get('run_label', '')} lacks {exc.args[0]}" diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index b3c3aa0..3041043 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -580,7 +580,17 @@ def _openclaw( "load": {"paths": [audit_plugin_root]}, "entries": {"shellbench-audit": {"enabled": True}}, }, - "tools": {"deny": ["message"]}, + "tools": { + "deny": ["message"], + "toolSearch": ( + { + "enabled": True, + "mode": run.openclaw_tool_search_mode, + } + if run.openclaw_tool_search_mode + else False + ), + }, } if servers: config["mcp"] = {"servers": servers} diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index cbe08be..c6d4595 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -32,6 +32,7 @@ class RunSpec: repetition: int expected_task_count: int run_date: str + openclaw_tool_search_mode: str | None = None def to_dict(self) -> dict[str, object]: return asdict(self) diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ccafe5e..ed71c0d 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -271,6 +271,11 @@ def _run_manifest( ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "openclaw_tool_search_mode": ( + run.openclaw_tool_search_mode + or os.environ.get("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE") + or None + ), "judge_reasoning_effort": os.environ.get( "SHELLBENCH_JUDGE_REASONING_EFFORT" ), @@ -358,6 +363,9 @@ def _runner_patch_hash() -> str: def build_run_spec(args: argparse.Namespace) -> RunSpec: harness = harness_by_name(args.harness) model = model_by_slug(args.model_slug) + openclaw_tool_search_mode = os.environ.get( + "SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE" + ) or None return RunSpec( run_label=args.run_label, harness=harness.name, @@ -369,6 +377,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, + openclaw_tool_search_mode=openclaw_tool_search_mode, ) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index b9855ff..24ab270 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -305,6 +305,7 @@ def __init__( self.dispatch_concurrency: dict[str, int] = {} self.dispatch_arguments: dict[str, list[str]] = {} self.dispatch_parity_validation: dict[str, str] = {} + self.dispatch_tool_search_mode: dict[str, str] = {} self.dispatch_qualification_family: dict[str, str] = {} self.dispatch_phase: dict[str, str] = {} self.dispatch_leaderboard_eligible: dict[str, str] = {} @@ -439,7 +440,8 @@ def run( leaderboard_eligible = remote_command[dispatch_marker + 17] exclusion_reason = remote_command[dispatch_marker + 18] parity_validation = remote_command[dispatch_marker + 20] - remote_run_args = remote_command[dispatch_marker + 21 :] + tool_search_mode = remote_command[dispatch_marker + 21] + remote_run_args = remote_command[dispatch_marker + 22 :] with self._dispatch_condition: attempt = self.dispatch_attempts.get(label, 0) + 1 self.dispatch_attempts[label] = attempt @@ -453,6 +455,7 @@ def run( self.dispatch_concurrency[label] = int(remote_run_args[10]) self.dispatch_arguments[label] = remote_run_args self.dispatch_parity_validation[label] = parity_validation + self.dispatch_tool_search_mode[label] = tool_search_mode self.dispatch_qualification_family[label] = qualification_family self.dispatch_phase[label] = run_phase self.dispatch_leaderboard_eligible[label] = leaderboard_eligible @@ -600,6 +603,20 @@ def test_controller_accepts_xhigh_reasoning_effort(tmp_path: Path) -> None: assert completed["judge_reasoning_effort"] == "high" +def test_controller_dispatches_openclaw_tool_search_mode(tmp_path: Path) -> None: + label = "openclaw-gpt55-high-full-2-r1-20260729" + run = _planned(_run_spec(label)) + run["openclaw_tool_search_mode"] = "code" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + + assert executor.dispatch_tool_search_mode[label] == "code" + + def test_controller_dispatches_targeted_repair_tasks_with_lineage( tmp_path: Path, ) -> None: @@ -1174,6 +1191,29 @@ def test_failed_run_is_preserved_and_suffixed_rerun_completes( assert (config.local_root / "raw" / f"{rerun}-final-artifacts.tar.gz").is_file() +def test_failed_run_preserves_openclaw_tool_search_mode_on_rerun( + tmp_path: Path, +) -> None: + base = "openclaw-gpt55-high-full-2-r1-20260729" + rerun = f"{base}-rerun1" + run = _planned(_run_spec(base)) + run["openclaw_tool_search_mode"] = "directory" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index, max_attempts=2) + executor = FakeExecutor( + config.local_root, + expected_counts={base: 1, rerun: 2}, + checkpoint_codes={base: 1}, + ) + + assert FleetController(config, executor=executor).run() == 0 + + runs = json.loads(run_index.read_text(encoding="utf-8"))["runs"] + assert runs[1]["openclaw_tool_search_mode"] == "directory" + assert executor.dispatch_tool_search_mode[rerun] == "directory" + + def test_resume_attaches_to_running_lease_without_redispatch(tmp_path: Path) -> None: label = "openclaw-gpt55-full-2-r1-20260727" run = _planned(_run_spec(label)) diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..ad0eccf 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -415,6 +415,7 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["parity_validated"] is False assert manifest["parity_validation"] is None assert manifest["legacy_parity_validated_claim"] is False + assert manifest["openclaw_tool_search_mode"] is None def test_run_manifest_excludes_r0_from_leaderboard( @@ -612,6 +613,29 @@ def test_run_spec_preserves_explicit_planned_identity() -> None: assert run.proxy_model_name == "planned-proxy-name" +def test_run_spec_normalizes_empty_tool_search_mode( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE", "") + + run = build_run_spec( + Namespace( + run_label="openclaw-tool-search-off", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260729", + ) + ) + + assert run.openclaw_tool_search_mode is None + + def test_harness_commands_preserve_canonical_model_identity() -> None: for harness in HARNESSES: run = RunSpec( @@ -679,6 +703,55 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command +def test_openclaw_harness_disables_tool_search_by_default() -> None: + run = RunSpec( + run_label="openclaw-tool-search-off", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=4, + run_date="20260729", + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert '"toolSearch":false' in command.setup_command + + +def test_openclaw_harness_configures_tool_search_code_mode() -> None: + run = RunSpec( + run_label="openclaw-tool-search-code", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=4, + run_date="20260729", + openclaw_tool_search_mode="code", + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert '"toolSearch":{"enabled":true,"mode":"code"}' in command.setup_command + + def test_openclaw_completion_probe_accepts_markerless_final_envelope( tmp_path: Path, ) -> None: From ebe901ef632be45029aaa9fe67030bd9d70d8782 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 30 Jul 2026 05:16:21 +0200 Subject: [PATCH 2/5] fix(native-eval): benchmark genuine OpenClaw code mode --- scripts/native_eval/fleet.py | 40 +++++---- scripts/native_eval/harnesses.py | 8 +- scripts/native_eval/models.py | 5 +- scripts/native_eval/run_job.py | 25 ++++-- tests/test_native_eval_fleet.py | 49 ++++++++--- tests/test_native_eval_runner.py | 134 +++++++++++++++++++++++++++++-- 6 files changed, 209 insertions(+), 52 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index feb70b2..e94aabe 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -19,7 +19,7 @@ from typing import Any, Protocol, Sequence from scripts.native_eval.checkpoint_loop import count_result_json -from scripts.native_eval.models import RunSpec +from scripts.native_eval.models import OPENCLAW_TOOL_MODES, RunSpec from scripts.native_eval.runtime import atomic_write_json, utc_now @@ -48,7 +48,6 @@ ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} -OPENCLAW_TOOL_SEARCH_MODES = {"directory", "code"} # Crabbox's coordinator release path retries five 60-second requests with # bounded backoff. Give it enough time to finish instead of leaking live AWS # leases after a verified export. @@ -442,17 +441,22 @@ def _validate_plan(self) -> None: f"{run.run_label} must set judge_reasoning_effort to " "low, medium, high, or xhigh" ) - tool_search_mode = str(entry.get("openclaw_tool_search_mode") or "") - if tool_search_mode: + if entry.get("openclaw_tool_search_mode"): + raise FleetError( + f"{run.run_label} uses retired openclaw_tool_search_mode; " + "replace it with openclaw_tool_mode" + ) + tool_mode = str(entry.get("openclaw_tool_mode") or "") + if tool_mode: if run.harness != "openclaw": raise FleetError( - f"{run.run_label} openclaw_tool_search_mode requires the " + f"{run.run_label} openclaw_tool_mode requires the " "OpenClaw harness" ) - if tool_search_mode not in OPENCLAW_TOOL_SEARCH_MODES: + if tool_mode not in OPENCLAW_TOOL_MODES: raise FleetError( - f"{run.run_label} must set openclaw_tool_search_mode to " - f"one of {sorted(OPENCLAW_TOOL_SEARCH_MODES)}" + f"{run.run_label} must set openclaw_tool_mode to " + f"one of {sorted(OPENCLAW_TOOL_MODES)}" ) def _prepare_inputs(self) -> None: @@ -971,7 +975,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: shift 9 parity_validated=$1 parity_validation_json=$2 -openclaw_tool_search_mode=$3 +openclaw_tool_mode=$3 shift 3 mkdir -p "$root/run-logs" stdout="$root/run-logs/$label.stdout.log" @@ -995,7 +999,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: "SHELLBENCH_EXCLUSION_REASON=$exclusion_reason" \ "SHELLBENCH_PARITY_VALIDATED=$parity_validated" \ "SHELLBENCH_PARITY_VALIDATION_JSON=$parity_validation_json" \ - "SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE=$openclaw_tool_search_mode" \ + "SHELLBENCH_OPENCLAW_TOOL_MODE=$openclaw_tool_mode" \ "$root/runner/scripts/native_eval/remote_run.sh" "$@" \ >"$stdout" 2>"$stderr" None: separators=(",", ":"), sort_keys=True, ) - openclaw_tool_search_mode = str( - entry.get("openclaw_tool_search_mode") or "" - ) + openclaw_tool_mode = str(entry.get("openclaw_tool_mode") or "") command = self._ssh_command( lease, [ @@ -1076,7 +1078,7 @@ def _dispatch(self, lease: Lease, run: RunSpec) -> None: exclusion_reason, str(self.config.parity_validated).lower(), parity_validation, - openclaw_tool_search_mode, + openclaw_tool_mode, *args, ], ) @@ -1482,9 +1484,7 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "run_label": label, "reasoning_effort": entry.get("reasoning_effort"), "judge_reasoning_effort": entry.get("judge_reasoning_effort"), - "openclaw_tool_search_mode": entry.get( - "openclaw_tool_search_mode" - ), + "openclaw_tool_mode": entry.get("openclaw_tool_mode"), "phase": entry.get("phase"), "qualification_family": entry.get("qualification_family"), "attempt": next_attempt, @@ -1500,7 +1500,7 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: "task_names", "rerun_of_canonical_run", "repair_classifications", - "openclaw_tool_search_mode", + "openclaw_tool_mode", ): if metadata_field in entry: rerun[metadata_field] = copy.deepcopy(entry[metadata_field]) @@ -1523,9 +1523,7 @@ def _run_spec(self, entry: dict[str, Any]) -> RunSpec: try: return RunSpec( **{field: entry[field] for field in RUN_SPEC_FIELDS}, - openclaw_tool_search_mode=entry.get( - "openclaw_tool_search_mode" - ), + openclaw_tool_mode=entry.get("openclaw_tool_mode"), ) except KeyError as exc: raise FleetError( diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 3041043..32fe434 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -582,12 +582,10 @@ def _openclaw( }, "tools": { "deny": ["message"], + "codeMode": run.openclaw_tool_mode == "code", "toolSearch": ( - { - "enabled": True, - "mode": run.openclaw_tool_search_mode, - } - if run.openclaw_tool_search_mode + {"enabled": True, "mode": "directory"} + if run.openclaw_tool_mode == "directory" else False ), }, diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index c6d4595..ca72d4e 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -5,6 +5,9 @@ from typing import Iterable +OPENCLAW_TOOL_MODES = frozenset({"direct", "directory", "code"}) + + @dataclass(frozen=True) class ModelSpec: slug: str @@ -32,7 +35,7 @@ class RunSpec: repetition: int expected_task_count: int run_date: str - openclaw_tool_search_mode: str | None = None + openclaw_tool_mode: str | None = None def to_dict(self) -> dict[str, object]: return asdict(self) diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index ed71c0d..82285c9 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -11,6 +11,7 @@ from typing import Any from scripts.native_eval.models import ( + OPENCLAW_TOOL_MODES, RunSpec, harness_by_name, model_by_slug, @@ -271,9 +272,9 @@ def _run_manifest( ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), - "openclaw_tool_search_mode": ( - run.openclaw_tool_search_mode - or os.environ.get("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE") + "openclaw_tool_mode": ( + run.openclaw_tool_mode + or os.environ.get("SHELLBENCH_OPENCLAW_TOOL_MODE") or None ), "judge_reasoning_effort": os.environ.get( @@ -363,9 +364,19 @@ def _runner_patch_hash() -> str: def build_run_spec(args: argparse.Namespace) -> RunSpec: harness = harness_by_name(args.harness) model = model_by_slug(args.model_slug) - openclaw_tool_search_mode = os.environ.get( - "SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE" - ) or None + if os.environ.get("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE"): + raise ValueError( + "SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE is retired; " + "use SHELLBENCH_OPENCLAW_TOOL_MODE" + ) + openclaw_tool_mode = os.environ.get("SHELLBENCH_OPENCLAW_TOOL_MODE") or None + if openclaw_tool_mode and harness.name != "openclaw": + raise ValueError("SHELLBENCH_OPENCLAW_TOOL_MODE requires the OpenClaw harness") + if openclaw_tool_mode and openclaw_tool_mode not in OPENCLAW_TOOL_MODES: + raise ValueError( + "SHELLBENCH_OPENCLAW_TOOL_MODE must be one of " + f"{sorted(OPENCLAW_TOOL_MODES)}" + ) return RunSpec( run_label=args.run_label, harness=harness.name, @@ -377,7 +388,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, - openclaw_tool_search_mode=openclaw_tool_search_mode, + openclaw_tool_mode=openclaw_tool_mode, ) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 24ab270..54481b2 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -305,7 +305,7 @@ def __init__( self.dispatch_concurrency: dict[str, int] = {} self.dispatch_arguments: dict[str, list[str]] = {} self.dispatch_parity_validation: dict[str, str] = {} - self.dispatch_tool_search_mode: dict[str, str] = {} + self.dispatch_tool_mode: dict[str, str] = {} self.dispatch_qualification_family: dict[str, str] = {} self.dispatch_phase: dict[str, str] = {} self.dispatch_leaderboard_eligible: dict[str, str] = {} @@ -440,7 +440,7 @@ def run( leaderboard_eligible = remote_command[dispatch_marker + 17] exclusion_reason = remote_command[dispatch_marker + 18] parity_validation = remote_command[dispatch_marker + 20] - tool_search_mode = remote_command[dispatch_marker + 21] + tool_mode = remote_command[dispatch_marker + 21] remote_run_args = remote_command[dispatch_marker + 22 :] with self._dispatch_condition: attempt = self.dispatch_attempts.get(label, 0) + 1 @@ -455,7 +455,7 @@ def run( self.dispatch_concurrency[label] = int(remote_run_args[10]) self.dispatch_arguments[label] = remote_run_args self.dispatch_parity_validation[label] = parity_validation - self.dispatch_tool_search_mode[label] = tool_search_mode + self.dispatch_tool_mode[label] = tool_mode self.dispatch_qualification_family[label] = qualification_family self.dispatch_phase[label] = run_phase self.dispatch_leaderboard_eligible[label] = leaderboard_eligible @@ -603,10 +603,10 @@ def test_controller_accepts_xhigh_reasoning_effort(tmp_path: Path) -> None: assert completed["judge_reasoning_effort"] == "high" -def test_controller_dispatches_openclaw_tool_search_mode(tmp_path: Path) -> None: +def test_controller_dispatches_openclaw_tool_mode(tmp_path: Path) -> None: label = "openclaw-gpt55-high-full-2-r1-20260729" run = _planned(_run_spec(label)) - run["openclaw_tool_search_mode"] = "code" + run["openclaw_tool_mode"] = "code" run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) config = _config(tmp_path, run_index) @@ -614,7 +614,36 @@ def test_controller_dispatches_openclaw_tool_search_mode(tmp_path: Path) -> None assert FleetController(config, executor=executor).run() == 0 - assert executor.dispatch_tool_search_mode[label] == "code" + assert executor.dispatch_tool_mode[label] == "code" + + +def test_controller_rejects_retired_openclaw_tool_search_mode( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-high-full-2-r1-20260729" + run = _planned(_run_spec(label)) + run["openclaw_tool_search_mode"] = "code" + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index) + + with pytest.raises(FleetError, match="replace it with openclaw_tool_mode"): + FleetController(config, executor=FakeExecutor(config.local_root)) + + +def test_controller_accepts_empty_retired_openclaw_tool_search_mode( + tmp_path: Path, +) -> None: + label = "openclaw-gpt55-high-full-2-r1-20260729" + run = _planned(_run_spec(label)) + run["openclaw_tool_search_mode"] = None + run_index = tmp_path / "manifests" / "run_index.json" + _write_index(run_index, [run]) + config = _config(tmp_path, run_index) + executor = FakeExecutor(config.local_root, expected_counts={label: 2}) + + assert FleetController(config, executor=executor).run() == 0 + assert executor.dispatch_tool_mode[label] == "" def test_controller_dispatches_targeted_repair_tasks_with_lineage( @@ -1191,13 +1220,13 @@ def test_failed_run_is_preserved_and_suffixed_rerun_completes( assert (config.local_root / "raw" / f"{rerun}-final-artifacts.tar.gz").is_file() -def test_failed_run_preserves_openclaw_tool_search_mode_on_rerun( +def test_failed_run_preserves_openclaw_tool_mode_on_rerun( tmp_path: Path, ) -> None: base = "openclaw-gpt55-high-full-2-r1-20260729" rerun = f"{base}-rerun1" run = _planned(_run_spec(base)) - run["openclaw_tool_search_mode"] = "directory" + run["openclaw_tool_mode"] = "directory" run_index = tmp_path / "manifests" / "run_index.json" _write_index(run_index, [run]) config = _config(tmp_path, run_index, max_attempts=2) @@ -1210,8 +1239,8 @@ def test_failed_run_preserves_openclaw_tool_search_mode_on_rerun( assert FleetController(config, executor=executor).run() == 0 runs = json.loads(run_index.read_text(encoding="utf-8"))["runs"] - assert runs[1]["openclaw_tool_search_mode"] == "directory" - assert executor.dispatch_tool_search_mode[rerun] == "directory" + assert runs[1]["openclaw_tool_mode"] == "directory" + assert executor.dispatch_tool_mode[rerun] == "directory" def test_resume_attaches_to_running_lease_without_redispatch(tmp_path: Path) -> None: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index ad0eccf..f268174 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -415,7 +415,7 @@ def test_run_manifest_records_native_audit_metadata( assert manifest["parity_validated"] is False assert manifest["parity_validation"] is None assert manifest["legacy_parity_validated_claim"] is False - assert manifest["openclaw_tool_search_mode"] is None + assert manifest["openclaw_tool_mode"] is None def test_run_manifest_excludes_r0_from_leaderboard( @@ -613,10 +613,10 @@ def test_run_spec_preserves_explicit_planned_identity() -> None: assert run.proxy_model_name == "planned-proxy-name" -def test_run_spec_normalizes_empty_tool_search_mode( +def test_run_spec_normalizes_empty_openclaw_tool_mode( monkeypatch, ) -> None: - monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE", "") + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_MODE", "") run = build_run_spec( Namespace( @@ -633,7 +633,97 @@ def test_run_spec_normalizes_empty_tool_search_mode( ) ) - assert run.openclaw_tool_search_mode is None + assert run.openclaw_tool_mode is None + + +def test_run_spec_rejects_retired_openclaw_tool_search_mode( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE", "code") + + with pytest.raises( + ValueError, + match="SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE is retired", + ): + build_run_spec( + Namespace( + run_label="openclaw-tool-search-code", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260729", + ) + ) + + +def test_run_spec_accepts_empty_retired_openclaw_tool_search_mode( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_SEARCH_MODE", "") + + run = build_run_spec( + Namespace( + run_label="openclaw-tool-search-off", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260730", + ) + ) + + assert run.openclaw_tool_mode is None + + +def test_run_spec_rejects_invalid_openclaw_tool_mode(monkeypatch) -> None: + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_MODE", "cod") + + with pytest.raises(ValueError, match="must be one of"): + build_run_spec( + Namespace( + run_label="openclaw-tool-mode-invalid", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260730", + ) + ) + + +def test_run_spec_rejects_openclaw_tool_mode_for_other_harness( + monkeypatch, +) -> None: + monkeypatch.setenv("SHELLBENCH_OPENCLAW_TOOL_MODE", "code") + + with pytest.raises(ValueError, match="requires the OpenClaw harness"): + build_run_spec( + Namespace( + run_label="codex-tool-mode-invalid", + harness="codex", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260730", + ) + ) def test_harness_commands_preserve_canonical_model_identity() -> None: @@ -703,7 +793,7 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "cat /logs/agent/codex-stderr.txt >&2" in command.run_command -def test_openclaw_harness_disables_tool_search_by_default() -> None: +def test_openclaw_harness_uses_direct_tools_by_default() -> None: run = RunSpec( run_label="openclaw-tool-search-off", harness="openclaw", @@ -724,10 +814,11 @@ def test_openclaw_harness_disables_tool_search_by_default() -> None: mcp_servers=(), ) + assert '"codeMode":false' in command.setup_command assert '"toolSearch":false' in command.setup_command -def test_openclaw_harness_configures_tool_search_code_mode() -> None: +def test_openclaw_harness_configures_code_mode() -> None: run = RunSpec( run_label="openclaw-tool-search-code", harness="openclaw", @@ -739,7 +830,33 @@ def test_openclaw_harness_configures_tool_search_code_mode() -> None: repetition=1, expected_task_count=4, run_date="20260729", - openclaw_tool_search_mode="code", + openclaw_tool_mode="code", + ) + + command = build_harness_command( + run, + proxy_url="http://host.docker.internal:4000", + proxy_key="local-proxy-key", + mcp_servers=(), + ) + + assert '"codeMode":true' in command.setup_command + assert '"toolSearch":false' in command.setup_command + + +def test_openclaw_harness_configures_tool_directory_mode() -> None: + run = RunSpec( + run_label="openclaw-tool-directory", + harness="openclaw", + harness_version="test", + model_slug="gpt56-sol", + model_id="gpt-5.6-sol", + provider="openai", + proxy_model_name="gpt-5.6-sol", + repetition=1, + expected_task_count=4, + run_date="20260729", + openclaw_tool_mode="directory", ) command = build_harness_command( @@ -749,7 +866,8 @@ def test_openclaw_harness_configures_tool_search_code_mode() -> None: mcp_servers=(), ) - assert '"toolSearch":{"enabled":true,"mode":"code"}' in command.setup_command + assert '"codeMode":false' in command.setup_command + assert '"toolSearch":{"enabled":true,"mode":"directory"}' in command.setup_command def test_openclaw_completion_probe_accepts_markerless_final_envelope( From 1f99e06d0badfb76d53db1fdb4541b26244f9d72 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 30 Jul 2026 05:20:57 +0200 Subject: [PATCH 3/5] test(native-eval): fix tool mode validation coverage --- tests/test_native_eval_fleet.py | 5 ++++- tests/test_native_eval_runner.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_native_eval_fleet.py b/tests/test_native_eval_fleet.py index 54481b2..9b92a83 100644 --- a/tests/test_native_eval_fleet.py +++ b/tests/test_native_eval_fleet.py @@ -628,7 +628,10 @@ def test_controller_rejects_retired_openclaw_tool_search_mode( config = _config(tmp_path, run_index) with pytest.raises(FleetError, match="replace it with openclaw_tool_mode"): - FleetController(config, executor=FakeExecutor(config.local_root)) + FleetController( + config, + executor=FakeExecutor(config.local_root, expected_counts={label: 2}), + ).run() def test_controller_accepts_empty_retired_openclaw_tool_search_mode( diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index f268174..6ed4b8f 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -9,6 +9,8 @@ from pathlib import Path from types import SimpleNamespace +import pytest + from scripts.native_eval.checkpoint_loop import ( count_result_json, next_checkpoint_sequence, From 7f6e2db65d6129f87cc1aac3e920288eb1fddc60 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 30 Jul 2026 06:12:36 +0200 Subject: [PATCH 4/5] fix(native-eval): validate public OpenClaw trajectories --- scripts/native_eval/fleet.py | 9 +- scripts/native_eval/harness_trajectories.py | 294 ++++- scripts/native_eval/harnesses.py | 715 +++++-------- scripts/native_eval/models.py | 12 + scripts/native_eval/plan.py | 1 - scripts/native_eval/proxy.py | 3 +- scripts/native_eval/run_job.py | 6 +- tests/test_native_eval_runner.py | 1066 +++++++++++-------- 8 files changed, 1176 insertions(+), 930 deletions(-) diff --git a/scripts/native_eval/fleet.py b/scripts/native_eval/fleet.py index e94aabe..7dc97d1 100644 --- a/scripts/native_eval/fleet.py +++ b/scripts/native_eval/fleet.py @@ -19,7 +19,11 @@ from typing import Any, Protocol, Sequence from scripts.native_eval.checkpoint_loop import count_result_json -from scripts.native_eval.models import OPENCLAW_TOOL_MODES, RunSpec +from scripts.native_eval.models import ( + OPENCLAW_TOOL_MODES, + REASONING_EFFORTS, + RunSpec, +) from scripts.native_eval.runtime import atomic_write_json, utc_now @@ -47,7 +51,6 @@ RERUN_STATUSES = {"failed", "lease_lost"} ACTIVE_RUN_STATUSES = {"leasing", "bootstrapping", "ready", "running"} CLEANUP_STATUSES = {"exported", "stop_pending"} -REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} # Crabbox's coordinator release path retries five 60-second requests with # bounded backoff. Give it enough time to finish instead of leaking live AWS # leases after a verified export. @@ -1482,7 +1485,6 @@ def _schedule_rerun(self, entry: dict[str, Any]) -> str | None: rerun = { **run.to_dict(), "run_label": label, - "reasoning_effort": entry.get("reasoning_effort"), "judge_reasoning_effort": entry.get("judge_reasoning_effort"), "openclaw_tool_mode": entry.get("openclaw_tool_mode"), "phase": entry.get("phase"), @@ -1523,6 +1525,7 @@ def _run_spec(self, entry: dict[str, Any]) -> RunSpec: try: return RunSpec( **{field: entry[field] for field in RUN_SPEC_FIELDS}, + reasoning_effort=entry.get("reasoning_effort"), openclaw_tool_mode=entry.get("openclaw_tool_mode"), ) except KeyError as exc: diff --git a/scripts/native_eval/harness_trajectories.py b/scripts/native_eval/harness_trajectories.py index 080b5b1..237e3c3 100644 --- a/scripts/native_eval/harness_trajectories.py +++ b/scripts/native_eval/harness_trajectories.py @@ -4,7 +4,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -12,6 +12,10 @@ OpenClawSessionTrace = tuple[str, Path, int, list[dict[str, Any]]] +_OPENCLAW_FATAL_EXPORT_WARNINGS = { + "cyclic-session-branch", + "incomplete-session-branch", +} def load_openclaw_envelope(path: Path) -> dict[str, Any] | None: @@ -41,7 +45,7 @@ def write_openclaw_trajectory( agent_dir: Path, ) -> dict[str, Any]: log_path = agent_dir / "openclaw.txt" - session_path = agent_dir / "openclaw.session.jsonl" + session_path = _openclaw_root_session_path(agent_dir) envelope = load_openclaw_envelope(log_path) meta = envelope.get("meta") if envelope else {} if not isinstance(meta, dict): @@ -63,8 +67,12 @@ def write_openclaw_trajectory( agent_dir, session_path, ) + export_metadata = _openclaw_export_metadata(session_path) root_records = session_tree[0][3] if session_tree else None session_models = _openclaw_session_models(session_path, records=root_records) + export_model = export_metadata.get("export_model") + if isinstance(export_model, str) and export_model: + session_models.add(export_model) log_models = _openclaw_log_models(log_path) child_models = { result["resolved_model"] @@ -94,9 +102,26 @@ def write_openclaw_trajectory( session_path, records=root_records, ) - if not terminal_event_seen and envelope is not None: + if export_metadata: + terminal_event_seen = export_metadata.get("export_terminal_event_seen") is True + elif not terminal_event_seen and envelope is not None: terminal_event_seen = _openclaw_envelope_terminal(envelope) - if session_tree_validation["session_tree_complete"] is not True: + visible_tools = export_metadata.get("export_visible_tools") + tool_mode_observed = ( + visible_tools == ["exec", "wait"] + and export_metadata.get("export_snapshot_used") is True + if run.openclaw_tool_mode == "code" and export_metadata + else True + ) + if ( + export_metadata.get("export_valid") is False + or ( + export_metadata + and export_metadata.get("export_terminal_status") != "success" + ) + or not tool_mode_observed + or session_tree_validation["session_tree_complete"] is not True + ): return _unavailable( session_path, runtime_model_name=runtime_model_name, @@ -104,9 +129,11 @@ def write_openclaw_trajectory( observed_models=observed_models, extra_validation={ "terminal_event_seen": terminal_event_seen, + "tool_mode_observed": tool_mode_observed, "parent_models": sorted(parent_models), "child_models": sorted(normalized_child_models), "log_models": sorted(normalized_log_models), + **export_metadata, **session_tree_validation, }, ) @@ -154,16 +181,20 @@ def write_openclaw_trajectory( "parent_models": sorted(parent_models), "child_models": sorted(normalized_child_models), "log_models": sorted(normalized_log_models), + "tool_mode_observed": tool_mode_observed, + **export_metadata, }, ) usage = ( _openclaw_session_tree_usage(session_tree) - if len(session_tree) > 1 + if export_metadata and session_tree else agent_meta.get("usage") ) - if not isinstance(usage, dict) and session_tree: - usage = _openclaw_session_tree_usage(session_tree) + if not isinstance(usage, dict): + usage = _openclaw_session_tree_usage(session_tree) if session_tree else None + if not isinstance(usage, dict): + usage = export_metadata.get("export_usage") if not isinstance(usage, dict): usage = _openclaw_session_usage(session_path, records=root_records) input_tokens = _int(usage.get("input")) @@ -200,6 +231,8 @@ def write_openclaw_trajectory( "stop_reason": _nested_string(meta, "completion", "stopReason"), "aborted": meta.get("aborted"), "terminal_event_seen": terminal_event_seen, + "tool_mode_observed": tool_mode_observed, + **export_metadata, **session_tree_validation, }, } @@ -218,6 +251,8 @@ def write_openclaw_trajectory( "log_models": sorted(normalized_log_models), "session_id": session_id, "terminal_event_seen": terminal_event_seen, + "tool_mode_observed": tool_mode_observed, + **export_metadata, **session_tree_validation, }, } @@ -682,6 +717,21 @@ def _openclaw_session_steps( def _openclaw_session_records(path: Path) -> list[dict[str, Any]]: if not path.is_file(): return [] + if path.name == "session-branch.json": + bundle = _openclaw_export_bundle(path) + if bundle is None: + return [] + _, branch, events = bundle + records = _openclaw_export_snapshot_records(events) + if not records: + entries = branch.get("entries") + records = ( + [entry for entry in entries if isinstance(entry, dict)] + if isinstance(entries, list) + else [] + ) + header = branch.get("header") + return [header, *records] if isinstance(header, dict) else records records: list[dict[str, Any]] = [] for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: @@ -693,6 +743,189 @@ def _openclaw_session_records(path: Path) -> list[dict[str, Any]]: return records +def _openclaw_export_bundle( + branch_path: Path, +) -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]] | None: + manifest = _load_json_object(branch_path.parent / "manifest.json") + branch = _load_json_object(branch_path) + if ( + manifest.get("traceSchema") != "openclaw-trajectory" + or manifest.get("schemaVersion") != 1 + or not isinstance(manifest.get("traceId"), str) + or not isinstance(manifest.get("sessionId"), str) + or not isinstance(manifest.get("sessionKey"), str) + or not isinstance(manifest.get("eventCount"), int) + or not isinstance(manifest.get("runtimeEventCount"), int) + or not isinstance(manifest.get("transcriptEventCount"), int) + or not isinstance(manifest.get("sourceFiles"), dict) + or not isinstance(branch.get("entries"), list) + ): + return None + warnings = manifest.get("warnings") + if isinstance(warnings, list) and any( + isinstance(warning, dict) + and warning.get("code") in _OPENCLAW_FATAL_EXPORT_WARNINGS + for warning in warnings + ): + return None + events: list[dict[str, Any]] = [] + events_path = branch_path.parent / "events.jsonl" + if events_path.is_file(): + for line in events_path.read_text(encoding="utf-8", errors="replace").splitlines(): + try: + event = json.loads(line) + except json.JSONDecodeError: + return None + if ( + not isinstance(event, dict) + or event.get("traceSchema") != "openclaw-trajectory" + or event.get("schemaVersion") != 1 + or event.get("traceId") != manifest["traceId"] + or event.get("sessionId") != manifest["sessionId"] + or event.get("sessionKey") != manifest["sessionKey"] + ): + return None + events.append(event) + if len(events) != manifest["eventCount"]: + return None + if ( + sum(event.get("source") == "runtime" for event in events) + != manifest["runtimeEventCount"] + or sum(event.get("source") == "transcript" for event in events) + != manifest["transcriptEventCount"] + ): + return None + return manifest, branch, events + + +def _openclaw_export_runtime_turn( + events: list[dict[str, Any]], +) -> tuple[dict[str, Any] | None, dict[str, Any] | None, dict[str, Any] | None]: + runtime_events = [event for event in events if event.get("source") == "runtime"] + terminal = next( + (event for event in reversed(runtime_events) if event.get("type") == "session.ended"), + None, + ) + run_id = terminal.get("runId") if isinstance(terminal, dict) else None + + def matching(event: dict[str, Any], event_type: str) -> bool: + if event.get("type") != event_type: + return False + return not isinstance(run_id, str) or event.get("runId") == run_id + + completion = next( + (event for event in reversed(runtime_events) if matching(event, "model.completed")), + None, + ) + context = next( + (event for event in reversed(runtime_events) if matching(event, "context.compiled")), + None, + ) + return terminal, completion, context + + +def _openclaw_export_snapshot_records( + events: list[dict[str, Any]], +) -> list[dict[str, Any]]: + _, completion, _ = _openclaw_export_runtime_turn(events) + data = completion.get("data") if isinstance(completion, dict) else None + if not isinstance(data, dict) or data.get("truncated") is True: + return [] + snapshot = data.get("messagesSnapshot") + if not isinstance(snapshot, list): + return [] + records: list[dict[str, Any]] = [] + for index, message in enumerate(snapshot): + if not isinstance(message, dict): + return [] + timestamp = message.get("timestamp") + if isinstance(timestamp, (int, float)): + timestamp = ( + datetime.fromtimestamp(timestamp / 1000, tz=timezone.utc) + .isoformat() + .replace("+00:00", "Z") + ) + records.append( + { + "type": "message", + "id": f"runtime-message-{index + 1}", + "timestamp": timestamp, + "message": message, + } + ) + roles = { + record["message"].get("role") + for record in records + if isinstance(record.get("message"), dict) + and isinstance(record["message"].get("role"), str) + } + if not {"user", "assistant"} <= roles: + return [] + return records + + +def _openclaw_export_metadata(path: Path) -> dict[str, Any]: + if path.name != "session-branch.json": + return {} + bundle = _openclaw_export_bundle(path) + if bundle is None: + return {"export_valid": False} + manifest, _, events = bundle + terminal, completion, context = _openclaw_export_runtime_turn(events) + completion_data = completion.get("data") if isinstance(completion, dict) else None + context_data = context.get("data") if isinstance(context, dict) else None + visible_tools = None + if isinstance(context_data, dict): + visible_tools = ( + context_data.get("providerVisibleTools") + if "providerVisibleTools" in context_data + else context_data.get("tools") + ) + visible_tool_names = ( + sorted( + { + tool["name"] + for tool in visible_tools + if isinstance(tool, dict) and isinstance(tool.get("name"), str) + } + ) + if isinstance(visible_tools, list) + else [] + ) + return { + "export_valid": True, + "export_event_count": len(events), + "export_runtime_event_count": manifest["runtimeEventCount"], + "export_transcript_event_count": manifest["transcriptEventCount"], + "export_terminal_event_seen": terminal is not None, + "export_terminal_status": ( + terminal.get("data", {}).get("status") + if isinstance(terminal, dict) and isinstance(terminal.get("data"), dict) + else None + ), + "export_snapshot_used": bool(_openclaw_export_snapshot_records(events)), + "export_visible_tools": visible_tool_names, + "export_model": completion.get("modelId") if isinstance(completion, dict) else None, + "export_usage": ( + completion_data.get("usage") if isinstance(completion_data, dict) else None + ), + } + + +def _openclaw_root_session_path(agent_dir: Path) -> Path: + legacy = agent_dir / "openclaw.session.jsonl" + if legacy.is_file(): + return legacy + entries, ambiguous_keys = _load_openclaw_session_index(agent_dir / "openclaw.sessions") + if "agent:main:main" in ambiguous_keys: + return legacy + indexed = entries.get("agent:main:main") + if indexed is None: + return legacy + entry, store_dir = indexed + return _resolve_archived_openclaw_session_path(store_dir, entry) or legacy + + _OPENCLAW_CANONICAL_SESSION_ENTRY_TYPES = { "message", "thinking_level_change", @@ -958,13 +1191,20 @@ def _openclaw_session_tree( seen_paths = {root_path.resolve()} while pending: parent_key, parent_path, parent_records = pending.pop(0) - for spawn in _openclaw_spawn_results( - parent_path, - records=parent_records, - ): - child_key = spawn.get("child_session_key") - if not child_key: - continue + transcript_children = [ + spawn["child_session_key"] + for spawn in _openclaw_spawn_results( + parent_path, + records=parent_records, + ) + if spawn.get("child_session_key") + ] + audited_children = sorted( + key + for key, event in audit.items() + if event.get("spawnedBy") == parent_key and key not in transcript_children + ) + for child_key in [*transcript_children, *audited_children]: accepted_spawn_count += 1 ancestor: str | None = parent_key while ancestor is not None and ancestor != child_key: @@ -998,6 +1238,7 @@ def _openclaw_session_tree( entry, path = deleted else: entry, store_dir = indexed + entry = {**entry, **audit.get(child_key, {})} path = _resolve_archived_openclaw_session_path(store_dir, entry) if path is None: deleted, deleted_error = _resolve_deleted_openclaw_session( @@ -1142,6 +1383,31 @@ def _load_openclaw_session_index( ambiguous_keys.add(key) continue entries[key] = (value, store_path.parent) + manifest_paths = sorted(archive.rglob("manifest.json")) if archive.is_dir() else [] + for manifest_path in manifest_paths: + manifest = _load_json_object(manifest_path) + key = manifest.get("sessionKey") + session_id = manifest.get("sessionId") + branch_path = manifest_path.parent / "session-branch.json" + if ( + not isinstance(key, str) + or not key.strip() + or not isinstance(session_id, str) + or not session_id.strip() + or not branch_path.is_file() + ): + continue + key = key.strip() + if key in entries: + ambiguous_keys.add(key) + continue + entries[key] = ( + { + "sessionId": session_id.strip(), + "sessionFile": branch_path.name, + }, + manifest_path.parent, + ) return entries, ambiguous_keys diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index 32fe434..b7fd110 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -15,373 +15,175 @@ NPM_BIN = TOOLCHAIN_ROOT / "npm-packages" / "node_modules" / ".bin" HERMES_BIN = TOOLCHAIN_ROOT / "home" / ".local" / "bin" -_OPENCLAW_COMPLETION_PROBE = """\ +_OPENCLAW_CHILD_EXPORTS_READY = """\ import json import pathlib import sys -try: - text = pathlib.Path(sys.argv[1]).read_text( - encoding="utf-8", - errors="replace", - ).strip() -except OSError: +path = pathlib.Path(sys.argv[1]) +if not path.is_file(): sys.exit(1) - -decoder = json.JSONDecoder() -for start in range(len(text) - 1, -1, -1): - if text[start] != "{": - continue +ready = False +spawned = {} +exported = {} +for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): try: - value, _ = decoder.raw_decode(text[start:]) - except (json.JSONDecodeError, ValueError): + event = json.loads(line) + except json.JSONDecodeError: continue + if not isinstance(event, dict): + continue + if event.get("type") == "audit_ready": + ready = True + continue + run_id = event.get("runId") + session_key = event.get("sessionKey") if ( - isinstance(value, dict) - and isinstance(value.get("payloads"), list) - and isinstance(value.get("meta"), dict) + not isinstance(run_id, str) + or not run_id.strip() + or not isinstance(session_key, str) + or not session_key.strip() ): - meta = value["meta"] - liveness = str(meta.get("livenessState") or "").lower() - if meta.get("yielded") is True or liveness in { - "active", - "paused", - "running", - "waiting", - }: - continue - completion = meta.get("completion") - stop_reason = ( - completion.get("stopReason") - if isinstance(completion, dict) - else meta.get("stopReason") - ) - visible_payload = any( - isinstance(item, dict) - and isinstance(item.get("text"), str) - and item["text"].strip() - and item.get("isReasoning") is not True - for item in value["payloads"] + continue + run_id = run_id.strip() + session_key = session_key.strip() + if event.get("type") == "subagent_spawned": + spawned[run_id] = session_key + elif event.get("type") == "subagent_exported": + exported[run_id] = event +if not ready: + sys.exit(1) +failed = [ + run_id + for run_id in sorted(spawned) + if run_id in exported and exported[run_id].get("exportOk") is not True +] +if failed: + for run_id in failed: + print( + f"child export failed: {spawned[run_id]} ({run_id})", + file=sys.stderr, ) - if visible_payload or stop_reason or meta.get("aborted") is True: - sys.exit(0) -sys.exit(1) + sys.exit(2) +if not spawned.keys() <= exported.keys(): + sys.exit(1) +for run_id in sorted(spawned): + output = exported[run_id].get("exportOutput") + if not isinstance(output, str) or not output.strip(): + sys.exit(2) + print(output.strip()) """ -# ShellBench pins OpenClaw 2026.7.1-2, whose runtime session contract is the -# sessions.json registry plus JSONL transcripts under each agent's sessions dir. -_OPENCLAW_SESSION_PROBE = """\ +_OPENCLAW_EXPORT_READY = """\ import json import pathlib import sys -sessions = pathlib.Path(sys.argv[1]) -audit_root = pathlib.Path(sys.argv[2]) +bundle = pathlib.Path(sys.argv[1]) +mode = sys.argv[2] +scope = sys.argv[3] try: - store = json.loads((sessions / "sessions.json").read_text(encoding="utf-8")) + manifest = json.loads((bundle / "manifest.json").read_text(encoding="utf-8")) + events = [ + json.loads(line) + for line in (bundle / "events.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] except (OSError, json.JSONDecodeError): sys.exit(1) -if not isinstance(store, dict): +if ( + manifest.get("traceSchema") != "openclaw-trajectory" + or manifest.get("schemaVersion") != 1 + or len(events) != manifest.get("eventCount") + or sum(event.get("source") == "runtime" for event in events) + != manifest.get("runtimeEventCount") + or sum(event.get("source") == "transcript" for event in events) + != manifest.get("transcriptEventCount") +): sys.exit(1) - -audit = {} -audit_path = audit_root / "sessions.jsonl" -if audit_path.is_file(): - for line in audit_path.read_text( - encoding="utf-8", - errors="replace", - ).splitlines(): - try: - event = json.loads(line) - except json.JSONDecodeError: - continue - key = event.get("sessionKey") if isinstance(event, dict) else None - if isinstance(key, str) and key.strip(): - audit.setdefault(key.strip(), {}).update(event) - -def session_path(key, entry): - def contained(root, candidate): - try: - return candidate.resolve().is_relative_to(root.resolve()) - except (OSError, RuntimeError): - return False - - session_file = entry.get("sessionFile") - if isinstance(session_file, str) and session_file.strip(): - path = pathlib.Path(session_file) - candidate = path if path.is_absolute() else sessions / path - if contained(sessions, candidate) and candidate.is_file(): - return candidate - session_id = entry.get("sessionId") - active = sessions / f"{session_id}.jsonl" if session_id else None - if active and active.is_file(): - return active - transcript = audit.get(key, {}).get("auditTranscript") - if isinstance(transcript, str) and transcript.strip(): - candidate = audit_root / transcript - return candidate if contained(audit_root, candidate) else None - return active - -def records_for(key, entry): - path = session_path(key, entry) - if path is None or not path.is_file(): - return [] - records = [] - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - try: - value = json.loads(line) - except json.JSONDecodeError: - continue - if isinstance(value, dict): - records.append(value) - return records - -def content_text(content): - if isinstance(content, str): - return content - if not isinstance(content, list): - return "" - return "".join( - part.get("text", "") - for part in content - if isinstance(part, dict) and isinstance(part.get("text"), str) +if any( + event.get("traceId") != manifest.get("traceId") + or event.get("sessionId") != manifest.get("sessionId") + or event.get("sessionKey") != manifest.get("sessionKey") + for event in events +): + sys.exit(1) +if any( + isinstance(warning, dict) + and warning.get("code") in {"cyclic-session-branch", "incomplete-session-branch"} + for warning in manifest.get("warnings", []) +): + sys.exit(1) +runtime = [event for event in events if event.get("source") == "runtime"] +terminal = next( + (event for event in reversed(runtime) if event.get("type") == "session.ended"), + None, +) +if terminal is None: + sys.exit(1) +run_id = terminal.get("runId") +terminal_data = terminal.get("data") +terminal_status = ( + terminal_data.get("status") if isinstance(terminal_data, dict) else None +) +if scope == "root" and terminal_status != "success": + sys.exit(1) +completion = next( + ( + event + for event in reversed(runtime) + if event.get("type") == "model.completed" + and (not isinstance(run_id, str) or event.get("runId") == run_id) + ), + None, +) +if completion is None: + if scope != "child" or terminal_status not in {"error", "interrupted"}: + sys.exit(1) +if mode == "code" and completion is not None: + completion_data = completion.get("data") + snapshot = ( + completion_data.get("messagesSnapshot") + if isinstance(completion_data, dict) + else None ) - -def active_records(records): - canonical_types = { - "message", - "thinking_level_change", - "model_change", - "compaction", - "reset", - "branch_summary", - "custom", - "custom_message", - "label", - "session_info", - } - nodes = {} - leaf = None - append_parent = None - explicit_update = False - invalid_leaf_ids = set() - - def text(value): - return value.strip() if isinstance(value, str) and value.strip() else None - - def resolve_parent(parent): - seen = set() - while parent is not None: - if parent in seen: - return parent - seen.add(parent) - node = nodes.get(parent) - if node is None or not node["is_leaf"]: - return parent - parent = node["parent"] - return None - - for record in records: - record_type = record.get("type") - canonical = record_type in canonical_types - explicit = "parentId" in record - if record_type == "session" or (not explicit and not canonical): - continue - record_id = text(record.get("id")) - if record_id is None: - continue - raw_parent = record.get("parentId") if explicit else leaf - parent = None if raw_parent is None else text(raw_parent) - if raw_parent is not None and parent is None: - continue - is_leaf = record_type == "leaf" - if is_leaf: - raw_target = record.get("targetId") - target = None if raw_target is None else text(raw_target) - raw_append = record.get("appendParentId", raw_target) - next_append = None if raw_append is None else text(raw_append) - if ( - (raw_target is not None and target is None) - or (raw_append is not None and next_append is None) - or record.get("appendMode") not in {None, "side"} - ): - continue - invalid = any( - ref is not None and (ref not in nodes or ref in invalid_leaf_ids) - for ref in (target, next_append) - ) - if invalid: - invalid_leaf_ids.add(record_id) - next_leaf = ... - next_append = append_parent - else: - parent = target - next_leaf = target - else: - if ( - explicit - and parent is not None - and parent not in nodes - and leaf is not None - ): - parent = leaf - elif ( - explicit - and record.get("appendMode") != "side" - and parent == append_parent - and leaf != append_parent - ): - parent = leaf - parent = resolve_parent(parent) - next_leaf = record_id if canonical and record.get("appendMode") != "side" else ... - next_append = record_id - node = { - "record": record, - "parent": parent, - "leaf": next_leaf, - "append": next_append, - "is_leaf": is_leaf, + if ( + not isinstance(completion_data, dict) + or completion_data.get("truncated") is True + or not isinstance(snapshot, list) + or not snapshot + or not all(isinstance(message, dict) for message in snapshot) + or not {"user", "assistant"} <= { + message.get("role") + for message in snapshot + if isinstance(message.get("role"), str) } - nodes[record_id] = node - append_parent = next_append - if next_leaf is not ...: - leaf = next_leaf - explicit_update = explicit_update or explicit - - if not explicit_update: - return records - if leaf is None: - return [] - selected = [] - seen = set() - current = leaf - while current is not None: - if current in seen: - return [] - seen.add(current) - node = nodes.get(current) - if node is None: - break - if not node["is_leaf"]: - selected.append(node["record"]) - current = node["parent"] - selected.reverse() - return selected - -def terminal(entry, records): - if str(entry.get("status") or "").lower() in { - "cancelled", - "deleted", - "error", - "failed", - "killed", - "reset", - "timeout", - }: - return bool(records) - for record in reversed(active_records(records)): - message = record.get("message") - if ( - record.get("type") != "message" - or not isinstance(message, dict) - ): - continue - if message.get("role") != "assistant": - return False - content = message.get("content") - parts = content if isinstance(content, list) else [] - text = content_text(content) - tools = [ - part - for part in parts - if isinstance(part, dict) and part.get("type") == "toolCall" - ] - return ( - bool(text.strip()) - and not tools - and str(message.get("stopReason") or "").lower() in {"end_turn", "stop"} - ) - return False - -def result_object(message): - details = message.get("details") - if isinstance(details, dict): - return details - text = content_text(message.get("content")).strip() - if not text: - return {} - try: - value = json.loads(text) - except json.JSONDecodeError: - decoder = json.JSONDecoder() - for start, char in enumerate(text): - if char != "{": - continue - try: - value, _ = decoder.raw_decode(text[start:]) - except json.JSONDecodeError: - continue - if isinstance(value, dict): - return value - return {} - return value if isinstance(value, dict) else {} - -def spawned_children(records): - pending = set() - children = [] - for record in active_records(records): - message = record.get("message") - if record.get("type") != "message" or not isinstance(message, dict): - continue - if message.get("role") == "assistant": - content = message.get("content") - if not isinstance(content, list): - continue - pending.update( - str(part.get("id")) - for part in content - if isinstance(part, dict) - and part.get("type") == "toolCall" - and part.get("name") == "sessions_spawn" - and part.get("id") - ) - continue - if message.get("role") != "toolResult": - continue - call_id = str(message.get("toolCallId") or "") - if call_id not in pending: - continue - pending.discard(call_id) - payload = result_object(message) - child = payload.get("childSessionKey") - if ( - str(payload.get("status") or "").lower() == "accepted" - and isinstance(child, str) - and child.strip() - ): - children.append(child.strip()) - return children - -root_key = "agent:main:main" -pending = [(root_key, None)] -seen = set() -while pending: - key, parent = pending.pop(0) - if key in seen: - continue - seen.add(key) - entry = store.get(key) - if not isinstance(entry, dict): - entry = audit.get(key) - if not isinstance(entry, dict): - sys.exit(1) - if parent and entry.get("spawnedBy") not in {None, "", parent}: + ): sys.exit(1) - records = records_for(key, entry) - if not terminal(entry, records): + context = next( + ( + event + for event in reversed(runtime) + if event.get("type") == "context.compiled" + and (not isinstance(run_id, str) or event.get("runId") == run_id) + ), + None, + ) + data = context.get("data") if isinstance(context, dict) else None + tools = None + if isinstance(data, dict): + tools = ( + data.get("providerVisibleTools") + if "providerVisibleTools" in data + else data.get("tools") + ) + names = sorted( + tool.get("name") + for tool in tools + if isinstance(tool, dict) and isinstance(tool.get("name"), str) + ) if isinstance(tools, list) else [] + if names != ["exec", "wait"]: sys.exit(1) - pending.extend((child, key) for child in spawned_children(records)) -sys.exit(0) """ _OPENCLAW_GATEWAY_PROBE = """\ @@ -408,40 +210,72 @@ def spawned_children(records): """ _OPENCLAW_AUDIT_PLUGIN = """\ +const crypto = require("node:crypto"); +const childProcess = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); -const zlib = require("node:zlib"); const auditRoot = path.join(process.env.HOME, ".openclaw", "shellbench-audit"); const eventPath = path.join(auditRoot, "sessions.jsonl"); +const workspace = process.env.SHELLBENCH_WORKSPACE || process.cwd(); +const exportsByRun = new Map(); function append(event) { fs.mkdirSync(auditRoot, { recursive: true }); fs.appendFileSync(eventPath, `${JSON.stringify(event)}\\n`, { mode: 0o600 }); } -function materializeTranscript(event) { - const source = event.sessionFile; - if (typeof source !== "string" || !source || !fs.existsSync(source)) { - return undefined; - } - const transcriptRoot = path.join(auditRoot, "transcripts"); - fs.mkdirSync(transcriptRoot, { recursive: true }); - const destination = path.join(transcriptRoot, `${event.sessionId}.jsonl`); - try { - const content = source.endsWith(".zst") - ? zlib.zstdDecompressSync(fs.readFileSync(source)) - : fs.readFileSync(source); - fs.writeFileSync(destination, content, { mode: 0o600 }); - return path.relative(auditRoot, destination); - } catch { - return undefined; +async function exportRun(runId, sessionKey) { + const cacheKey = `${runId}\\0${sessionKey}`; + const cached = exportsByRun.get(cacheKey); + if (cached) { + return await cached; } + const digest = crypto.createHash("sha256").update(cacheKey).digest("hex").slice(0, 16); + const output = `shellbench-child-${digest}`; + const pending = new Promise((resolve) => { + childProcess.execFile( + "openclaw", + [ + "sessions", + "export-trajectory", + "--session-key", + sessionKey, + "--workspace", + workspace, + "--output", + output, + "--json", + ], + { + cwd: workspace, + encoding: "utf8", + env: process.env, + timeout: 120000, + }, + (error, _stdout, stderr) => { + resolve({ + exportOk: !error, + exportOutput: output, + exportStatus: + typeof error?.code === "number" ? error.code : error ? null : 0, + exportError: error ? String(error.message || error) : undefined, + exportStderr: + typeof stderr === "string" && stderr.trim() + ? stderr.trim().slice(-4000) + : undefined, + }); + }, + ); + }); + exportsByRun.set(cacheKey, pending); + return await pending; } module.exports = { id: "shellbench-audit", register(api) { + append({ type: "audit_ready" }); api.on("subagent_spawned", (event, ctx) => { append({ type: "subagent_spawned", @@ -450,32 +284,28 @@ def spawned_children(records): runId: event.runId, }); }); - api.on("subagent_ended", (event, ctx) => { + api.on("subagent_progress", async (event, ctx) => { + if (event.phase !== "ended") { + return; + } + const exported = await exportRun(event.runId, event.childSessionKey); append({ - type: "subagent_ended", - sessionKey: event.targetSessionKey, + type: "subagent_exported", + sessionKey: event.childSessionKey, spawnedBy: ctx.requesterSessionKey, runId: event.runId, status: event.outcome, - reason: event.reason, - }); - }); - api.on("session_start", (event) => { - append({ - type: "session_start", - sessionKey: event.sessionKey, - sessionId: event.sessionId, + ...exported, }); }); - api.on("session_end", (event) => { - const auditTranscript = materializeTranscript(event); + api.on("subagent_ended", (event, ctx) => { append({ - type: "session_end", - sessionKey: event.sessionKey, - sessionId: event.sessionId, + type: "subagent_ended", + sessionKey: event.targetSessionKey, + spawnedBy: ctx.requesterSessionKey, + runId: event.runId, + status: event.outcome, reason: event.reason, - transcriptArchived: event.transcriptArchived, - auditTranscript, }); }); }, @@ -532,6 +362,8 @@ def _openclaw( ) -> HarnessCommand: provider = "openai" model = f"{provider}/{run.model_id}" + thinking = run.reasoning_effort or "off" + tool_mode = run.openclaw_tool_mode or "direct" home = "/tmp/shellbench-openclaw" audit_plugin_root = f"{home}/.openclaw/shellbench-audit" gateway_token = secrets.token_urlsafe(32) @@ -553,7 +385,8 @@ def _openclaw( "workspace": ".", "skipBootstrap": True, "model": {"primary": model}, - "subagents": {"model": model}, + "thinkingDefault": thinking, + "subagents": {"model": model, "thinking": thinking}, } }, "gateway": { @@ -593,19 +426,19 @@ def _openclaw( if servers: config["mcp"] = {"servers": servers} config_json = shlex.quote(json.dumps(config, separators=(",", ":"))) - completion_probe = shlex.quote(_OPENCLAW_COMPLETION_PROBE) - session_probe = shlex.quote(_OPENCLAW_SESSION_PROBE) + child_exports_ready = shlex.quote(_OPENCLAW_CHILD_EXPORTS_READY) + export_ready = shlex.quote(_OPENCLAW_EXPORT_READY) gateway_probe = shlex.quote(_OPENCLAW_GATEWAY_PROBE) audit_plugin = shlex.quote(_OPENCLAW_AUDIT_PLUGIN) audit_manifest = shlex.quote(json.dumps(_OPENCLAW_AUDIT_PLUGIN_MANIFEST, separators=(",", ":"))) setup = ( - f"export PATH={_base_path()}; export HOME={home}; " + f"set -eu; export PATH={_base_path()}; export HOME={home}; " 'rm -rf "$HOME"; mkdir -p "$HOME/.openclaw/shellbench-audit"; ' f'printf %s {audit_plugin} > "$HOME/.openclaw/shellbench-audit/index.cjs"; ' f"printf %s {audit_manifest} " '> "$HOME/.openclaw/shellbench-audit/openclaw.plugin.json"; ' f'printf %s {config_json} > "$HOME/.openclaw/openclaw.json"; ' - "openclaw setup --baseline --skip-bootstrap --workspace . " + "openclaw setup --baseline --workspace . " ">/logs/agent/setup.log 2>&1; " "rm -f AGENTS.md BOOTSTRAP.md HEARTBEAT.md IDENTITY.md " "SOUL.md TOOLS.md USER.md; " @@ -614,6 +447,7 @@ def _openclaw( run_command = ( f"export PATH={_base_path()}; export HOME={home}; " f"export OPENCLAW_GATEWAY_TOKEN={shlex.quote(gateway_token)}; " + "export SHELLBENCH_WORKSPACE=\"$PWD\"; " "log=/logs/agent/openclaw.txt; " "gateway_log=/logs/agent/openclaw-gateway.txt; " 'openclaw gateway --port 18789 >"$gateway_log" 2>&1 & gateway_pid=$!; ' @@ -624,93 +458,66 @@ def _openclaw( 'kill "$gateway_pid" 2>/dev/null || true; ' 'wait "$gateway_pid" 2>/dev/null || true; ' 'cat "$gateway_log" >&2; exit 70; fi; ' - "openclaw agent --json --agent main --thinking off " + "openclaw agent --json --agent main " + f"--thinking {shlex.quote(thinking)} " f"--model {shlex.quote(model)} " '--message "$(cat /tmp/shellbench-instruction.md)" ' - '>"$log" 2>&1 /dev/null; do ' - f'if python3 -c {completion_probe} "$log"; then ' - 'sleep 1; kill "$pid" 2>/dev/null || true; sleep 1; ' - 'kill -KILL "$pid" 2>/dev/null || true; ' - 'wait "$pid" 2>/dev/null || true; reaped=1; break; ' - "fi; sleep 1; done; " - 'if [ "$reaped" -ne 1 ]; then wait "$pid"; status=$?; fi; ' - 'session_ready=0; if [ "$status" -eq 0 ]; then ' - "for _ in $(seq 1 60); do " - f"if python3 -c {session_probe} " - '"$HOME/.openclaw/agents/main/sessions" ' - '"$HOME/.openclaw/shellbench-audit"; then ' - "session_ready=1; break; fi; " - 'if ! kill -0 "$gateway_pid" 2>/dev/null; then status=70; break; fi; ' + '>"$log" 2>&1 >"$log" 2>&1 ' + f"&& python3 -c {export_ready} " + '".openclaw/trajectory-exports/shellbench-root" ' + f"{shlex.quote(tool_mode)} root; then " + "export_ok=1; break; fi; sleep 1; done; " + 'if [ "$export_ok" -ne 1 ]; then ' + "echo 'OpenClaw root trajectory export failed' >>\"$log\"; status=71; fi; " + "fi; " + 'if [ "$status" -eq 0 ]; then ' + "child_wait=0; while true; do " + f"python3 -c {child_exports_ready} " + '"$HOME/.openclaw/shellbench-audit/sessions.jsonl" ' + ">/tmp/shellbench-openclaw-child-exports.txt 2>>\"$log\"; " + "child_state=$?; " + 'if [ "$child_state" -eq 0 ]; then break; fi; ' + 'if [ "$child_state" -eq 2 ]; then status=71; break; fi; ' + 'if ! kill -0 "$gateway_pid" 2>/dev/null; then ' + "echo 'OpenClaw gateway exited while child exports were pending' " + '>>"$log"; status=70; break; fi; ' + "child_wait=$((child_wait + 1)); " + 'if [ "$child_wait" -ge 300 ]; then ' + "echo 'OpenClaw child trajectory exports did not settle within 300s' " + '>>"$log"; status=71; break; fi; ' "sleep 1; done; fi; " - 'if [ "$status" -eq 0 ] && [ "$session_ready" -ne 1 ]; then ' - "echo 'OpenClaw terminal session evidence did not stabilize within 60 seconds' " - '>>"$log"; status=71; fi; ' + 'if [ "$status" -eq 0 ]; then ' + "while IFS= read -r output; do " + '[ -n "$output" ] || continue; ' + f"if ! python3 -c {export_ready} " + '".openclaw/trajectory-exports/$output" ' + f"{shlex.quote(tool_mode)} child; then " + 'echo "OpenClaw child trajectory validation failed: $output" ' + '>>"$log"; status=71; break; fi; ' + "done /dev/null || true; ' 'wait "$gateway_pid" 2>/dev/null || true; ' 'cat "$gateway_log" >>"$log"; cat "$log"; exit "$status"' ) - # Keep this archive path aligned with the pinned OpenClaw 2026.7.1-2 - # sessions.json/JSONL contract used by the completion probe above. cleanup = ( "python3 - <<'PY'\n" - "import json, pathlib, shutil\n" - "p=pathlib.Path('/logs/agent/openclaw.txt')\n" - "agents=pathlib.Path('/tmp/shellbench-openclaw/.openclaw/agents')\n" - "sessions=agents/'main'/'sessions'\n" + "import pathlib, shutil\n" "archive=pathlib.Path('/logs/agent/openclaw.sessions')\n" "audit=pathlib.Path('/tmp/shellbench-openclaw/.openclaw/shellbench-audit')\n" - "if agents.is_dir():\n" - " for agent in agents.iterdir():\n" - " source=agent/'sessions'\n" - " if not source.is_dir(): continue\n" - " target=archive if agent.name=='main' else archive/'agents'/agent.name\n" - " shutil.copytree(source, target, dirs_exist_ok=True)\n" + "exports=pathlib.Path('.openclaw/trajectory-exports')\n" + "if exports.is_dir():\n" + " shutil.copytree(exports, archive/'exports', dirs_exist_ok=True)\n" "if audit.is_dir():\n" " shutil.copytree(audit, archive/'audit', dirs_exist_ok=True)\n" "if archive.is_dir():\n" " for item in [archive, *archive.rglob('*')]:\n" " item.chmod(0o755 if item.is_dir() else 0o644)\n" - "sources=[]\n" - "try:\n" - " raw=p.read_text(encoding='utf-8', errors='replace').strip()\n" - " dec=json.JSONDecoder(); d=None\n" - " for start in range(len(raw)-1, -1, -1):\n" - " if raw[start] != '{': continue\n" - " try: candidate, _ = dec.raw_decode(raw[start:])\n" - " except (json.JSONDecodeError, ValueError): continue\n" - " if isinstance(candidate, dict) and " - "isinstance(candidate.get('meta'), dict):\n" - " d=candidate; break\n" - " if d:\n" - " src=((d.get('meta') or {}).get('agentMeta') or {}).get('sessionFile')\n" - " if isinstance(src, str) and src: sources.append(src)\n" - "except Exception:\n" - " pass\n" - "if sessions.is_dir():\n" - " try:\n" - " store=json.loads((sessions/'sessions.json').read_text())\n" - " entry=store.get('agent:main:main') if isinstance(store, dict) else None\n" - " if isinstance(entry, dict):\n" - " session_file=entry.get('sessionFile')\n" - " if isinstance(session_file, str) and session_file:\n" - " sources.append(session_file)\n" - " if entry.get('sessionId'):\n" - " sources.append(str(sessions/f\"{entry['sessionId']}.jsonl\"))\n" - " except Exception:\n" - " pass\n" - "source=None\n" - "for src in sources:\n" - " candidate=pathlib.Path(src)\n" - " if not candidate.is_absolute(): candidate=sessions/candidate\n" - " if candidate.is_file():\n" - " source=candidate\n" - " break\n" - "if source and source.is_file():\n" - " destination=pathlib.Path('/logs/agent/openclaw.session.jsonl')\n" - " shutil.copy2(source, destination)\n" - " destination.chmod(0o644)\n" "PY" ) return HarnessCommand( diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index ca72d4e..d0dee80 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -6,6 +6,7 @@ OPENCLAW_TOOL_MODES = frozenset({"direct", "directory", "code"}) +REASONING_EFFORTS = frozenset({"low", "medium", "high", "xhigh"}) @dataclass(frozen=True) @@ -35,8 +36,18 @@ class RunSpec: repetition: int expected_task_count: int run_date: str + reasoning_effort: str | None = None openclaw_tool_mode: str | None = None + def __post_init__(self) -> None: + if ( + self.reasoning_effort is not None + and self.reasoning_effort not in REASONING_EFFORTS + ): + raise ValueError( + "reasoning_effort must be low, medium, high, xhigh, or None" + ) + def to_dict(self) -> dict[str, object]: return asdict(self) @@ -166,6 +177,7 @@ def build_matrix_plan( repetition=repetition, expected_task_count=expected_task_count, run_date=stamp, + reasoning_effort=reasoning_effort, ) ) return plan diff --git a/scripts/native_eval/plan.py b/scripts/native_eval/plan.py index 372aec0..5f330aa 100644 --- a/scripts/native_eval/plan.py +++ b/scripts/native_eval/plan.py @@ -86,7 +86,6 @@ def write_run_index( entries = [ { **run.to_dict(), - "reasoning_effort": reasoning_effort, "judge_model_id": judge_model_id, "judge_reasoning_effort": judge_reasoning_effort, "phase": phase, diff --git a/scripts/native_eval/proxy.py b/scripts/native_eval/proxy.py index fe14142..be1bfae 100644 --- a/scripts/native_eval/proxy.py +++ b/scripts/native_eval/proxy.py @@ -4,11 +4,10 @@ import os from pathlib import Path -from scripts.native_eval.models import LITELLM_VERSION, MODELS +from scripts.native_eval.models import LITELLM_VERSION, MODELS, REASONING_EFFORTS JUDGE_PROXY_MODEL_NAME = "shellbench-judge" -REASONING_EFFORTS = {"low", "medium", "high", "xhigh"} def write_proxy_config(path: Path) -> None: diff --git a/scripts/native_eval/run_job.py b/scripts/native_eval/run_job.py index 82285c9..8c85c9a 100644 --- a/scripts/native_eval/run_job.py +++ b/scripts/native_eval/run_job.py @@ -271,7 +271,10 @@ def _run_manifest( "SHELLBENCH_HARBOR_REFERENCE_COMMIT" ), "judge_model_id": os.environ.get("SHELLBENCH_JUDGE_MODEL_ID"), - "reasoning_effort": os.environ.get("SHELLBENCH_REASONING_EFFORT"), + "reasoning_effort": ( + run.reasoning_effort + or os.environ.get("SHELLBENCH_REASONING_EFFORT") + ), "openclaw_tool_mode": ( run.openclaw_tool_mode or os.environ.get("SHELLBENCH_OPENCLAW_TOOL_MODE") @@ -388,6 +391,7 @@ def build_run_spec(args: argparse.Namespace) -> RunSpec: repetition=args.repetition, expected_task_count=args.expected_task_count, run_date=args.run_date, + reasoning_effort=os.environ.get("SHELLBENCH_REASONING_EFFORT") or None, openclaw_tool_mode=openclaw_tool_mode, ) diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6ed4b8f..cefe32f 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -16,8 +16,8 @@ next_checkpoint_sequence, ) from scripts.native_eval.harnesses import ( - _OPENCLAW_COMPLETION_PROBE, - _OPENCLAW_SESSION_PROBE, + _OPENCLAW_CHILD_EXPORTS_READY, + _OPENCLAW_EXPORT_READY, build_harness_command, ) from scripts.native_eval.harness_trajectories import ( @@ -52,13 +52,18 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: - plan = build_matrix_plan(116, run_date="20260727") + plan = build_matrix_plan( + 116, + run_date="20260727", + reasoning_effort="high", + ) assert len(plan) == 96 assert len({run.run_label for run in plan}) == 96 assert {run.harness for run in plan} == {harness.name for harness in HARNESSES} assert {run.model_slug for run in plan} == {model.slug for model in MODELS} assert {run.repetition for run in plan} == {1, 2, 3} + assert {run.reasoning_effort for run in plan} == {"high"} def test_run_index_records_agent_and_judge_reasoning( @@ -615,6 +620,47 @@ def test_run_spec_preserves_explicit_planned_identity() -> None: assert run.proxy_model_name == "planned-proxy-name" +def test_run_spec_preserves_planned_reasoning_effort(monkeypatch) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "high") + + run = build_run_spec( + Namespace( + run_label="openclaw-reasoning-high", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260730", + ) + ) + + assert run.reasoning_effort == "high" + + +def test_run_spec_rejects_invalid_reasoning_effort(monkeypatch) -> None: + monkeypatch.setenv("SHELLBENCH_REASONING_EFFORT", "extreme") + + with pytest.raises(ValueError, match="reasoning_effort must be"): + build_run_spec( + Namespace( + run_label="openclaw-reasoning-invalid", + harness="openclaw", + harness_version="planned-version", + model_slug="gpt55", + model_id="gpt-5.5", + model_provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=3, + run_date="20260730", + ) + ) + + def test_run_spec_normalizes_empty_openclaw_tool_mode( monkeypatch, ) -> None: @@ -741,6 +787,7 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: repetition=1, expected_task_count=116, run_date="20260727", + reasoning_effort="high", ) command = build_harness_command( run, @@ -756,36 +803,49 @@ def test_harness_commands_preserve_canonical_model_identity() -> None: assert "OPENROUTER_API_KEY" not in command.env assert 'exit "$status"' in command.run_command if harness.name == "openclaw": - assert "ended with stopReason=" not in command.run_command assert "python3 -c" in command.run_command - assert "--thinking off" in command.run_command + assert "--thinking high" in command.run_command assert "openclaw gateway --port 18789" in command.run_command assert "127.0.0.1:18789/readyz" in command.run_command assert "openclaw agent --local" not in command.run_command + assert "kill -KILL" not in command.run_command + assert "openclaw sessions export-trajectory" in command.run_command + assert '--session-key "agent:main:main"' in command.run_command assert "OPENCLAW_GATEWAY_TOKEN" in command.env assert len(command.env["OPENCLAW_GATEWAY_TOKEN"]) >= 32 - assert "seq 1 60" in command.run_command + assert "seq 1 10" in command.run_command assert "status=71" in command.run_command - assert "setup --baseline --skip-bootstrap" in command.setup_command + assert command.setup_command.startswith("set -eu;") + assert "setup --baseline --workspace ." in command.setup_command + assert "--skip-bootstrap" not in command.setup_command assert "rm -f AGENTS.md BOOTSTRAP.md HEARTBEAT.md" in (command.setup_command) assert '"skipBootstrap":true' in command.setup_command assert '"primary":"openai/claude-opus-5"' in command.setup_command - assert '"subagents":{"model":"openai/claude-opus-5"}' in (command.setup_command) + assert '"thinkingDefault":"high"' in command.setup_command + assert ( + '"subagents":{"model":"openai/claude-opus-5","thinking":"high"}' + in command.setup_command + ) assert '"agentRuntime":{"id":"openclaw"}' in command.setup_command assert '"shellbench-audit":{"enabled":true}' in command.setup_command assert "openclaw.plugin.json" in command.setup_command assert '"configSchema":{"type":"object"' in command.setup_command assert 'api.on("subagent_spawned"' in command.setup_command + assert 'api.on("subagent_progress"' in command.setup_command assert 'api.on("subagent_ended"' in command.setup_command - assert 'api.on("session_end"' in command.setup_command - assert "zstdDecompressSync" in command.setup_command - assert "catch {" in command.setup_command + assert "event.phase !==" in command.setup_command + assert "execFile(" in command.setup_command + assert '"export-trajectory"' in command.setup_command + assert "exportsByRun" in command.setup_command + assert "zstdDecompressSync" not in command.setup_command + assert "shellbench-openclaw-child-exports.txt" in command.run_command + assert 'kill -0 "$gateway_pid"' in command.run_command + assert "child_wait" in command.run_command + assert "did not settle within 300s" in command.run_command assert "item.chmod(0o755 if item.is_dir() else 0o644)" in (command.cleanup_command) - assert "for agent in agents.iterdir()" in command.cleanup_command assert "archive/'audit'" in command.cleanup_command - assert "sources.append(str(sessions" in command.cleanup_command - assert "max(candidates" not in command.cleanup_command - assert "destination.chmod(0o644)" in command.cleanup_command + assert "archive/'exports'" in command.cleanup_command + assert "sessions.json" not in command.cleanup_command if harness.name == "hermes": assert '"delegation":{"max_iterations":50' in command.setup_command assert '"provider":"custom:shellbench"' in command.setup_command @@ -872,325 +932,106 @@ def test_openclaw_harness_configures_tool_directory_mode() -> None: assert '"toolSearch":{"enabled":true,"mode":"directory"}' in command.setup_command -def test_openclaw_completion_probe_accepts_markerless_final_envelope( +def test_openclaw_child_exports_wait_for_every_spawned_session( tmp_path: Path, ) -> None: - log_path = tmp_path / "openclaw.txt" - log_path.write_text( - "debug preamble\n" - + json.dumps( - { - "payloads": [{"text": "done"}], - "meta": {"aborted": False}, - } - ), - encoding="utf-8", - ) - - completed = subprocess.run( - [sys.executable, "-c", _OPENCLAW_COMPLETION_PROBE, str(log_path)], - check=False, - ) - - assert completed.returncode == 0 - - -def test_openclaw_completion_probe_rejects_partial_log(tmp_path: Path) -> None: - log_path = tmp_path / "openclaw.txt" - log_path.write_text( - 'debug preamble\n{"payloads":[{"text":"still writing"}]', - encoding="utf-8", - ) - - completed = subprocess.run( - [sys.executable, "-c", _OPENCLAW_COMPLETION_PROBE, str(log_path)], - check=False, - ) - - assert completed.returncode == 1 - - -def test_openclaw_completion_probe_rejects_paused_yielded_envelope( - tmp_path: Path, -) -> None: - log_path = tmp_path / "openclaw.txt" - log_path.write_text( - json.dumps( - { - "payloads": [{"text": "waiting for child"}], - "meta": { - "aborted": False, - "livenessState": "paused", - "yielded": True, - "stopReason": "end_turn", - }, - } - ), - encoding="utf-8", - ) - - completed = subprocess.run( - [sys.executable, "-c", _OPENCLAW_COMPLETION_PROBE, str(log_path)], - check=False, - ) - - assert completed.returncode == 1 - - -def test_openclaw_session_probe_requires_terminal_accepted_tree( - tmp_path: Path, -) -> None: - sessions = tmp_path / "sessions" - sessions.mkdir() - audit = tmp_path / "audit" - audit.mkdir() - child_key = "agent:main:subagent:child" - root_records = [ - {"type": "session", "id": "root"}, - { - "type": "message", - "message": {"role": "user", "content": "delegate"}, - }, - { - "type": "message", - "message": { - "role": "assistant", - "content": [ - { - "type": "toolCall", - "id": "spawn-1", - "name": "sessions_spawn", - "arguments": {"task": "inspect"}, - } - ], - "stopReason": "toolUse", - }, - }, - { - "type": "message", - "message": { - "role": "toolResult", - "toolCallId": "spawn-1", - "content": json.dumps( - { - "status": "accepted", - "childSessionKey": child_key, - } - ), - }, - }, - { - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": "done"}], - "stopReason": "stop", - }, - }, - ] - child_records = [ - {"type": "session", "id": "child"}, - { - "type": "message", - "message": {"role": "user", "content": "inspect"}, - }, - { - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": "inspected"}], - "stopReason": "stop", - }, - }, - ] - (sessions / "root.jsonl").write_text( - "\n".join(json.dumps(record) for record in root_records) + "\n", - encoding="utf-8", - ) - (sessions / "child.jsonl").write_text( - "\n".join(json.dumps(record) for record in child_records) + "\n", - encoding="utf-8", - ) - store = { - "agent:main:main": { - "sessionId": "root", - "sessionFile": "root.jsonl", - }, - child_key: { - "sessionId": "child", - "sessionFile": "child.jsonl", - "spawnedBy": "agent:main:main", - "status": "done", - }, - } - (sessions / "sessions.json").write_text(json.dumps(store), encoding="utf-8") - - complete = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) + audit = tmp_path / "sessions.jsonl" + child = "agent:main:subagent:child" + nested = "agent:main:subagent:nested" - assert complete.returncode == 0 + def run_probe() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_CHILD_EXPORTS_READY, + str(audit), + ], + check=False, + capture_output=True, + text=True, + ) - root_records[3]["message"]["content"] = "spawn accepted: " + json.dumps( - { - "status": "accepted", - "childSessionKey": child_key, - } - ) - (sessions / "root.jsonl").write_text( - "\n".join(json.dumps(record) for record in root_records) + "\n", + assert run_probe().returncode == 1 + audit.write_text( + json.dumps({"type": "audit_ready"}) + "\n", encoding="utf-8", ) - embedded_result = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert embedded_result.returncode == 0 - - store.pop(child_key) - (sessions / "sessions.json").write_text(json.dumps(store), encoding="utf-8") - missing_child = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert missing_child.returncode == 1 + assert run_probe().returncode == 0 + with audit.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "type": "subagent_spawned", + "sessionKey": child, + "runId": "run-child", + } + ) + + "\n" + ) + assert run_probe().returncode == 1 - (audit / "sessions.jsonl").write_text( - json.dumps( + with audit.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "type": "subagent_spawned", + "sessionKey": nested, + "runId": "run-nested", + } + ) + + "\n" + ) + handle.write( + json.dumps( { - "type": "subagent_ended", - "sessionKey": child_key, - "spawnedBy": "agent:main:main", - "status": "failed", + "type": "subagent_exported", + "sessionKey": child, + "runId": "run-child", + "exportOk": True, + "exportOutput": "shellbench-child-one", } ) - + "\n", - encoding="utf-8", - ) - failed_without_transcript = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert failed_without_transcript.returncode == 1 - - (audit / "transcripts").mkdir() - (audit / "transcripts" / "child.jsonl").write_text( - "\n".join(json.dumps(record) for record in child_records) + "\n", - encoding="utf-8", - ) - (audit / "sessions.jsonl").write_text( - "\n".join( - [ - json.dumps( - { - "type": "subagent_spawned", - "sessionKey": child_key, - "spawnedBy": "agent:main:main", - } - ), - json.dumps( - { - "type": "session_end", - "sessionKey": child_key, - "sessionId": "child", - "status": "done", - "auditTranscript": "transcripts/child.jsonl", - } - ), - ] + + "\n" ) - + "\n", - encoding="utf-8", - ) - audited_child = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert audited_child.returncode == 0 - - store[child_key] = { - "sessionId": "child", - "sessionFile": "missing-child.jsonl", - "spawnedBy": "agent:main:main", - "status": "done", - } - (sessions / "sessions.json").write_text(json.dumps(store), encoding="utf-8") - stale_index_with_audit = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert stale_index_with_audit.returncode == 0 + assert run_probe().returncode == 1 - (audit / "transcripts" / "child.jsonl").write_text( - "\n".join(json.dumps(record) for record in child_records[:2]) + "\n", - encoding="utf-8", - ) - for terminal_status in ("deleted", "error", "reset"): - with (audit / "sessions.jsonl").open("a", encoding="utf-8") as audit_file: - audit_file.write( - json.dumps( - { - "type": "subagent_ended", - "sessionKey": child_key, - "status": terminal_status, - } - ) - + "\n" + with audit.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "type": "subagent_exported", + "sessionKey": nested, + "runId": "run-nested", + "exportOk": True, + "exportOutput": "shellbench-child-two", + } ) - failed_child = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, + + "\n" ) + completed = run_probe() + assert completed.returncode == 0 + assert completed.stdout.splitlines() == [ + "shellbench-child-one", + "shellbench-child-two", + ] - assert failed_child.returncode == 0 + with audit.open("a", encoding="utf-8") as handle: + handle.write( + json.dumps( + { + "type": "subagent_exported", + "sessionKey": nested, + "runId": "run-nested", + "exportOk": False, + "exportOutput": "shellbench-child-two", + } + ) + + "\n" + ) + failed = run_probe() + assert failed.returncode == 2 + assert nested in failed.stderr def test_openclaw_terminal_rejects_an_unanswered_latest_user_turn( @@ -1223,142 +1064,6 @@ def test_openclaw_terminal_rejects_an_unanswered_latest_user_turn( assert _openclaw_session_terminal(session) is False - sessions = tmp_path / "sessions" - sessions.mkdir() - (sessions / "root.jsonl").write_text(session.read_text(encoding="utf-8")) - (sessions / "sessions.json").write_text( - json.dumps( - { - "agent:main:main": { - "sessionId": "root", - "sessionFile": "root.jsonl", - } - } - ), - encoding="utf-8", - ) - audit = tmp_path / "audit" - audit.mkdir() - - probe = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert probe.returncode == 1 - - -def test_openclaw_session_probe_rejects_audit_paths_outside_its_root( - tmp_path: Path, -) -> None: - sessions = tmp_path / "sessions" - sessions.mkdir() - audit = tmp_path / "audit" - audit.mkdir() - outside = tmp_path / "outside.jsonl" - child_key = "agent:main:subagent:child" - root_records = [ - {"type": "session", "id": "root"}, - { - "type": "message", - "message": {"role": "user", "content": "delegate"}, - }, - { - "type": "message", - "message": { - "role": "assistant", - "content": [ - { - "type": "toolCall", - "id": "spawn-1", - "name": "sessions_spawn", - "arguments": {"task": "inspect"}, - } - ], - "stopReason": "toolUse", - }, - }, - { - "type": "message", - "message": { - "role": "toolResult", - "toolCallId": "spawn-1", - "content": json.dumps({"status": "accepted", "childSessionKey": child_key}), - }, - }, - { - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": "done"}], - "stopReason": "stop", - }, - }, - ] - terminal_records = [ - {"type": "session", "id": "outside"}, - { - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": "forged"}], - "stopReason": "stop", - }, - }, - ] - (sessions / "root.jsonl").write_text( - "\n".join(json.dumps(record) for record in root_records) + "\n", - encoding="utf-8", - ) - outside.write_text( - "\n".join(json.dumps(record) for record in terminal_records) + "\n", - encoding="utf-8", - ) - (sessions / "sessions.json").write_text( - json.dumps( - { - "agent:main:main": { - "sessionId": "root", - "sessionFile": "root.jsonl", - } - } - ), - encoding="utf-8", - ) - (audit / "sessions.jsonl").write_text( - json.dumps( - { - "type": "session_end", - "sessionKey": child_key, - "sessionId": "child", - "spawnedBy": "agent:main:main", - "status": "done", - "auditTranscript": "../outside.jsonl", - } - ) - + "\n", - encoding="utf-8", - ) - - probe = subprocess.run( - [ - sys.executable, - "-c", - _OPENCLAW_SESSION_PROBE, - str(sessions), - str(audit), - ], - check=False, - ) - - assert probe.returncode == 1 - def test_openclaw_archived_session_path_preserves_safe_subdirectories( tmp_path: Path, @@ -2228,6 +1933,457 @@ def test_openclaw_session_without_envelope_converts_to_atif( assert fallback_trajectory["final_metrics"]["total_cached_tokens"] == 20 +def test_openclaw_exported_trajectory_bundle_converts_to_atif( + tmp_path: Path, +) -> None: + agent_dir = tmp_path / "agent" + bundle = agent_dir / "openclaw.sessions" / "exports" / "shellbench-root" + bundle.mkdir(parents=True) + user = {"role": "user", "content": "do the task", "timestamp": 1_753_833_602_000} + exec_call = { + "role": "assistant", + "model": "gpt-5.5", + "timestamp": 1_753_833_603_000, + "content": [ + { + "type": "toolCall", + "id": "call-exec", + "name": "exec", + "arguments": {"code": "await tools.shell({cmd: 'pwd'})"}, + } + ], + "usage": {"input": 12, "output": 3, "cacheRead": 5}, + "stopReason": "toolUse", + } + exec_result = { + "role": "toolResult", + "toolCallId": "call-exec", + "timestamp": 1_753_833_604_000, + "content": [{"type": "text", "text": "code completed"}], + } + nested_call = { + "role": "assistant", + "timestamp": 1_753_833_604_100, + "content": [ + { + "type": "toolCall", + "id": "tool_search_code:call-exec:shell:1", + "name": "shell", + "arguments": {"cmd": "pwd"}, + } + ], + "stopReason": "toolUse", + } + nested_result = { + "role": "toolResult", + "toolCallId": "tool_search_code:call-exec:shell:1", + "timestamp": 1_753_833_604_200, + "content": [{"type": "text", "text": "/app"}], + } + final = { + "role": "assistant", + "model": "gpt-5.5", + "timestamp": 1_753_833_605_000, + "content": [{"type": "text", "text": "done"}], + "usage": {"input": 4, "output": 2, "cacheRead": 1}, + "stopReason": "stop", + } + persisted_messages = [user, exec_call, exec_result, final] + snapshot = [user, exec_call, exec_result, nested_call, nested_result, final] + records = [ + { + "type": "message", + "id": f"entry-{index}", + "timestamp": f"2026-07-30T00:00:0{index}Z", + "message": message, + } + for index, message in enumerate(persisted_messages, start=1) + ] + runtime_events = [ + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "traceId": "session-export-123", + "source": "runtime", + "type": "context.compiled", + "ts": "2026-07-30T00:00:01Z", + "seq": 1, + "sourceSeq": 1, + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "runId": "run-1", + "data": { + "tools": [{"name": "shell"}, {"name": "read"}], + "providerVisibleTools": [{"name": "exec"}, {"name": "wait"}], + }, + }, + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "traceId": "session-export-123", + "source": "runtime", + "type": "model.completed", + "ts": "2026-07-30T00:00:03Z", + "seq": 2, + "sourceSeq": 2, + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "runId": "run-1", + "provider": "openai", + "modelId": "gpt-5.5", + "data": {"usage": {"input": 12, "output": 3, "cacheRead": 5}}, + }, + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "traceId": "session-export-123", + "source": "runtime", + "type": "model.completed", + "ts": "2026-07-30T00:00:05Z", + "seq": 3, + "sourceSeq": 3, + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "runId": "run-1", + "provider": "openai", + "modelId": "gpt-5.5", + "data": { + "usage": {"input": 4, "output": 2, "cacheRead": 1}, + "messagesSnapshot": snapshot, + }, + }, + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "traceId": "session-export-123", + "source": "runtime", + "type": "session.ended", + "ts": "2026-07-30T00:00:06Z", + "seq": 4, + "sourceSeq": 4, + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "runId": "run-1", + "data": {"status": "success"}, + }, + ] + transcript_events = [ + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "traceId": "session-export-123", + "source": "transcript", + "type": "assistant.message", + "ts": record["timestamp"], + "seq": 4 + index, + "sourceSeq": index, + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "entryId": record["id"], + "data": {"message": record["message"]}, + } + for index, record in enumerate(records, start=1) + ] + events = [*runtime_events, *transcript_events] + (bundle / "manifest.json").write_text( + json.dumps( + { + "traceSchema": "openclaw-trajectory", + "schemaVersion": 1, + "generatedAt": "2026-07-30T00:00:07Z", + "traceId": "session-export-123", + "sessionId": "session-export-123", + "sessionKey": "agent:main:main", + "workspaceDir": "$WORKSPACE_DIR", + "leafId": "entry-4", + "eventCount": len(events), + "runtimeEventCount": len(runtime_events), + "transcriptEventCount": len(transcript_events), + "sourceFiles": {"session": "$OPENCLAW_STATE/session"}, + } + ), + encoding="utf-8", + ) + (bundle / "session-branch.json").write_text( + json.dumps( + { + "header": { + "type": "session", + "id": "session-export-123", + "timestamp": "2026-07-30T00:00:00Z", + "cwd": "/app", + }, + "leafId": "entry-4", + "entries": records, + } + ), + encoding="utf-8", + ) + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + (agent_dir / "openclaw.txt").write_text( + "[provider-transport-fetch] [model-fetch] response " + "provider=openai api=openai-responses model=gpt-5.5 status=200\n", + encoding="utf-8", + ) + run = RunSpec( + run_label="openclaw-gpt55-code", + harness="openclaw", + harness_version="test", + model_slug="gpt55", + model_id="gpt-5.5", + provider="openai", + proxy_model_name="gpt-5.5", + repetition=1, + expected_task_count=1, + run_date="20260730", + openclaw_tool_mode="code", + ) + + metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + trajectory = json.loads((agent_dir / "trajectory.json").read_text()) + + assert metadata["trajectory_status"] == "real" + assert metadata["trajectory_validation"]["trace_fidelity"] == "session" + assert metadata["trajectory_validation"]["session_tree_session_count"] == 1 + assert metadata["trajectory_validation"]["export_snapshot_used"] is True + assert metadata["trajectory_validation"]["export_visible_tools"] == ["exec", "wait"] + assert metadata["trajectory_validation"]["tool_mode_observed"] is True + assert trajectory["session_id"] == "session-export-123" + assert trajectory["steps"][1]["tool_calls"][0]["function_name"] == "exec" + assert trajectory["steps"][2]["tool_calls"][0]["function_name"] == "shell" + assert trajectory["steps"][2]["observation"]["results"][0]["content"] == "/app" + assert trajectory["steps"][-1]["message"] == "done" + assert trajectory["final_metrics"]["total_prompt_tokens"] == 22 + assert trajectory["final_metrics"]["total_completion_tokens"] == 5 + assert trajectory["final_metrics"]["total_cached_tokens"] == 6 + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 0 + ) + latest_completion = runtime_events[2] + terminal_event = runtime_events[3] + terminal_event["data"] = {"status": "error"} + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + failed_root_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert failed_root_metadata["trajectory_status"] == "unavailable" + assert failed_root_metadata["trajectory_validation"]["export_terminal_status"] == "error" + terminal_event["data"] = {"status": "success"} + latest_completion["data"]["messagesSnapshot"] = [] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + latest_completion["data"]["messagesSnapshot"] = snapshot + latest_completion["data"].pop("messagesSnapshot") + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + incomplete_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert incomplete_metadata["trajectory_status"] == "unavailable" + assert incomplete_metadata["trajectory_validation"]["export_snapshot_used"] is False + assert incomplete_metadata["trajectory_validation"]["tool_mode_observed"] is False + latest_completion["data"]["messagesSnapshot"] = snapshot + runtime_events[0]["data"]["providerVisibleTools"] = [{"name": "exec"}] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + runtime_events[0]["data"]["tools"] = [{"name": "exec"}, {"name": "wait"}] + runtime_events[0]["data"]["providerVisibleTools"] = [] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + empty_visible_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert empty_visible_metadata["trajectory_status"] == "unavailable" + assert empty_visible_metadata["trajectory_validation"]["export_visible_tools"] == [] + assert empty_visible_metadata["trajectory_validation"]["tool_mode_observed"] is False + runtime_events[0]["data"]["tools"] = [{"name": "shell"}, {"name": "read"}] + runtime_events[0]["data"]["providerVisibleTools"] = [ + {"name": "exec"}, + {"name": "wait"}, + ] + latest_completion["traceId"] = "stale-trace" + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + invalid_trace_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert invalid_trace_metadata["trajectory_status"] == "unavailable" + assert invalid_trace_metadata["trajectory_validation"]["export_valid"] is False + latest_completion["traceId"] = "session-export-123" + terminal_only_events = [ + { + **event, + "data": {"status": "error"}, + } + for event in events + if event["type"] == "session.ended" + ] + manifest = json.loads((bundle / "manifest.json").read_text()) + manifest["eventCount"] = len(terminal_only_events) + len(transcript_events) + manifest["runtimeEventCount"] = len(terminal_only_events) + (bundle / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + (bundle / "events.jsonl").write_text( + "".join( + json.dumps(event) + "\n" + for event in [*terminal_only_events, *transcript_events] + ), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "child", + ], + check=False, + ).returncode + == 0 + ) + terminal_only_events[0]["data"] = {"status": "success"} + (bundle / "events.jsonl").write_text( + "".join( + json.dumps(event) + "\n" + for event in [*terminal_only_events, *transcript_events] + ), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "child", + ], + check=False, + ).returncode + == 1 + ) + terminal_only_events[0]["data"] = {"status": "error"} + (bundle / "events.jsonl").write_text( + "".join( + json.dumps(event) + "\n" + for event in [*terminal_only_events, *transcript_events] + ), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + + def test_openclaw_transport_model_mismatch_invalidates_identity(tmp_path: Path) -> None: agent_dir = tmp_path / "agent" agent_dir.mkdir() From f5b31cb1260b631b811bf0ede9f4eba51a94b98c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 30 Jul 2026 06:35:47 +0200 Subject: [PATCH 5/5] fix(native-eval): validate observed code-mode surface --- scripts/native_eval/harness_trajectories.py | 77 ++++++- scripts/native_eval/harnesses.py | 17 +- tests/test_native_eval_runner.py | 224 +++++++++++++++++++- 3 files changed, 306 insertions(+), 12 deletions(-) diff --git a/scripts/native_eval/harness_trajectories.py b/scripts/native_eval/harness_trajectories.py index 237e3c3..e54acc1 100644 --- a/scripts/native_eval/harness_trajectories.py +++ b/scripts/native_eval/harness_trajectories.py @@ -16,6 +16,12 @@ "cyclic-session-branch", "incomplete-session-branch", } +_OPENCLAW_CODE_MODE_HIDDEN_TOOLS = { + "tool_call", + "tool_describe", + "tool_search", + "tool_search_code", +} def load_openclaw_envelope(path: Path) -> dict[str, Any] | None: @@ -108,7 +114,10 @@ def write_openclaw_trajectory( terminal_event_seen = _openclaw_envelope_terminal(envelope) visible_tools = export_metadata.get("export_visible_tools") tool_mode_observed = ( - visible_tools == ["exec", "wait"] + export_metadata.get("export_provider_visible_tools_recorded") is True + and isinstance(visible_tools, list) + and {"exec", "wait"} <= set(visible_tools) + and not _OPENCLAW_CODE_MODE_HIDDEN_TOOLS & set(visible_tools) and export_metadata.get("export_snapshot_used") is True if run.openclaw_tool_mode == "code" and export_metadata else True @@ -864,6 +873,59 @@ def _openclaw_export_snapshot_records( return records +def _openclaw_export_snapshot_diagnostics( + records: list[dict[str, Any]], +) -> dict[str, Any]: + pending: set[str] = set() + tool_call_count = 0 + tool_result_count = 0 + tool_error_count = 0 + terminal_outcome = "unknown" + for record in records: + message = record.get("message") + if record.get("type") != "message" or not isinstance(message, dict): + continue + role = message.get("role") + if role == "assistant": + text, tools = _openclaw_assistant_content(message.get("content")) + for tool in tools: + call_id = tool.get("tool_call_id") + if isinstance(call_id, str) and call_id: + pending.add(call_id) + tool_call_count += 1 + terminal_assistant_text = ( + bool(text.strip()) + and not tools + and str(message.get("stopReason") or "").lower() in {"end_turn", "stop"} + ) + terminal_outcome = ( + "assistant_text" + if terminal_assistant_text + else "unresolved_tool_call" + if tools + else "unknown" + ) + elif role == "toolResult": + tool_result_count += 1 + call_id = message.get("toolCallId") + if isinstance(call_id, str) and call_id: + pending.discard(call_id) + is_error = message.get("isError") is True + if is_error: + tool_error_count += 1 + terminal_outcome = "tool_error" if is_error else "resolved_tool_result" + elif role in {"user"}: + terminal_outcome = "unknown" + outcome = "unresolved_tool_call" if pending else terminal_outcome + return { + "export_snapshot_outcome": outcome, + "export_snapshot_tool_call_count": tool_call_count, + "export_snapshot_tool_result_count": tool_result_count, + "export_snapshot_tool_error_count": tool_error_count, + "export_snapshot_pending_tool_call_count": len(pending), + } + + def _openclaw_export_metadata(path: Path) -> dict[str, Any]: if path.name != "session-branch.json": return {} @@ -874,6 +936,14 @@ def _openclaw_export_metadata(path: Path) -> dict[str, Any]: terminal, completion, context = _openclaw_export_runtime_turn(events) completion_data = completion.get("data") if isinstance(completion, dict) else None context_data = context.get("data") if isinstance(context, dict) else None + snapshot_recorded = ( + isinstance(completion_data, dict) and "messagesSnapshot" in completion_data + ) + snapshot_records = _openclaw_export_snapshot_records(events) + snapshot_diagnostics = _openclaw_export_snapshot_diagnostics(snapshot_records) + provider_visible_tools_recorded = ( + isinstance(context_data, dict) and "providerVisibleTools" in context_data + ) visible_tools = None if isinstance(context_data, dict): visible_tools = ( @@ -903,7 +973,10 @@ def _openclaw_export_metadata(path: Path) -> dict[str, Any]: if isinstance(terminal, dict) and isinstance(terminal.get("data"), dict) else None ), - "export_snapshot_used": bool(_openclaw_export_snapshot_records(events)), + "export_snapshot_recorded": snapshot_recorded, + "export_snapshot_used": bool(snapshot_records), + **snapshot_diagnostics, + "export_provider_visible_tools_recorded": provider_visible_tools_recorded, "export_visible_tools": visible_tool_names, "export_model": completion.get("modelId") if isinstance(completion, dict) else None, "export_usage": ( diff --git a/scripts/native_eval/harnesses.py b/scripts/native_eval/harnesses.py index b7fd110..aa068ff 100644 --- a/scripts/native_eval/harnesses.py +++ b/scripts/native_eval/harnesses.py @@ -170,19 +170,20 @@ None, ) data = context.get("data") if isinstance(context, dict) else None - tools = None - if isinstance(data, dict): - tools = ( - data.get("providerVisibleTools") - if "providerVisibleTools" in data - else data.get("tools") - ) + if not isinstance(data, dict) or "providerVisibleTools" not in data: + sys.exit(1) + tools = data.get("providerVisibleTools") names = sorted( tool.get("name") for tool in tools if isinstance(tool, dict) and isinstance(tool.get("name"), str) ) if isinstance(tools, list) else [] - if names != ["exec", "wait"]: + name_set = set(names) + if ( + not {"exec", "wait"} <= name_set + or name_set + & {"tool_search_code", "tool_search", "tool_describe", "tool_call"} + ): sys.exit(1) """ diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index cefe32f..a140828 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -2014,7 +2014,13 @@ def test_openclaw_exported_trajectory_bundle_converts_to_atif( "runId": "run-1", "data": { "tools": [{"name": "shell"}, {"name": "read"}], - "providerVisibleTools": [{"name": "exec"}, {"name": "wait"}], + "providerVisibleTools": [ + {"name": "computer"}, + {"name": "exec"}, + {"name": "image"}, + {"name": "sessions_yield"}, + {"name": "wait"}, + ], }, }, { @@ -2152,8 +2158,24 @@ def test_openclaw_exported_trajectory_bundle_converts_to_atif( assert metadata["trajectory_status"] == "real" assert metadata["trajectory_validation"]["trace_fidelity"] == "session" assert metadata["trajectory_validation"]["session_tree_session_count"] == 1 + assert metadata["trajectory_validation"]["export_snapshot_recorded"] is True assert metadata["trajectory_validation"]["export_snapshot_used"] is True - assert metadata["trajectory_validation"]["export_visible_tools"] == ["exec", "wait"] + assert metadata["trajectory_validation"]["export_snapshot_outcome"] == "assistant_text" + assert metadata["trajectory_validation"]["export_snapshot_tool_call_count"] == 2 + assert metadata["trajectory_validation"]["export_snapshot_tool_result_count"] == 2 + assert metadata["trajectory_validation"]["export_snapshot_tool_error_count"] == 0 + assert ( + metadata["trajectory_validation"]["export_snapshot_pending_tool_call_count"] + == 0 + ) + assert metadata["trajectory_validation"]["export_provider_visible_tools_recorded"] is True + assert metadata["trajectory_validation"]["export_visible_tools"] == [ + "computer", + "exec", + "image", + "sessions_yield", + "wait", + ] assert metadata["trajectory_validation"]["tool_mode_observed"] is True assert trajectory["session_id"] == "session-export-123" assert trajectory["steps"][1]["tool_calls"][0]["function_name"] == "exec" @@ -2177,8 +2199,139 @@ def test_openclaw_exported_trajectory_bundle_converts_to_atif( ).returncode == 0 ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "direct", + "root", + ], + check=False, + ).returncode + == 0 + ) latest_completion = runtime_events[2] terminal_event = runtime_events[3] + bookkeeping = {"role": "system", "content": "runtime bookkeeping"} + latest_completion["data"]["messagesSnapshot"] = [*snapshot, bookkeeping] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + bookkeeping_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert bookkeeping_metadata["trajectory_status"] == "real" + assert ( + bookkeeping_metadata["trajectory_validation"]["export_snapshot_outcome"] + == "assistant_text" + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 0 + ) + failed_exec_result = {**exec_result, "isError": True} + latest_completion["data"]["messagesSnapshot"] = [ + user, + exec_call, + failed_exec_result, + bookkeeping, + ] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + failed_tool_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert failed_tool_metadata["trajectory_status"] == "real" + assert ( + failed_tool_metadata["trajectory_validation"]["export_snapshot_outcome"] + == "tool_error" + ) + assert ( + failed_tool_metadata["trajectory_validation"]["export_snapshot_tool_error_count"] + == 1 + ) + for tool_mode in ("direct", "code"): + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + tool_mode, + "root", + ], + check=False, + ).returncode + == 0 + ) + latest_completion["data"]["messagesSnapshot"] = [ + user, + exec_call, + failed_exec_result, + final, + bookkeeping, + ] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + recovered_tool_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert recovered_tool_metadata["trajectory_status"] == "real" + assert ( + recovered_tool_metadata["trajectory_validation"]["export_snapshot_outcome"] + == "assistant_text" + ) + assert ( + recovered_tool_metadata["trajectory_validation"]["export_snapshot_tool_error_count"] + == 1 + ) + latest_completion["data"]["messagesSnapshot"] = [user, exec_call, bookkeeping] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + unresolved_tool_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert unresolved_tool_metadata["trajectory_status"] == "real" + assert ( + unresolved_tool_metadata["trajectory_validation"]["export_snapshot_outcome"] + == "unresolved_tool_call" + ) + assert ( + unresolved_tool_metadata["trajectory_validation"][ + "export_snapshot_pending_tool_call_count" + ] + == 1 + ) + latest_completion["data"]["messagesSnapshot"] = snapshot terminal_event["data"] = {"status": "error"} (bundle / "events.jsonl").write_text( "".join(json.dumps(event) + "\n" for event in events), @@ -2271,9 +2424,76 @@ def test_openclaw_exported_trajectory_bundle_converts_to_atif( agent_dir, ) assert empty_visible_metadata["trajectory_status"] == "unavailable" + assert ( + empty_visible_metadata["trajectory_validation"][ + "export_provider_visible_tools_recorded" + ] + is True + ) assert empty_visible_metadata["trajectory_validation"]["export_visible_tools"] == [] assert empty_visible_metadata["trajectory_validation"]["tool_mode_observed"] is False runtime_events[0]["data"]["tools"] = [{"name": "shell"}, {"name": "read"}] + runtime_events[0]["data"].pop("providerVisibleTools") + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + missing_visible_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert missing_visible_metadata["trajectory_status"] == "unavailable" + assert ( + missing_visible_metadata["trajectory_validation"][ + "export_provider_visible_tools_recorded" + ] + is False + ) + runtime_events[0]["data"]["providerVisibleTools"] = [ + {"name": "exec"}, + {"name": "tool_search"}, + {"name": "wait"}, + ] + (bundle / "events.jsonl").write_text( + "".join(json.dumps(event) + "\n" for event in events), + encoding="utf-8", + ) + assert ( + subprocess.run( + [ + sys.executable, + "-c", + _OPENCLAW_EXPORT_READY, + str(bundle), + "code", + "root", + ], + check=False, + ).returncode + == 1 + ) + visible_search_metadata = write_agent_trajectory( + _trajectory_task(tmp_path, "do the task"), + run, + agent_dir, + ) + assert visible_search_metadata["trajectory_status"] == "unavailable" + assert visible_search_metadata["trajectory_validation"]["tool_mode_observed"] is False runtime_events[0]["data"]["providerVisibleTools"] = [ {"name": "exec"}, {"name": "wait"},