diff --git a/scripts/native_eval/models.py b/scripts/native_eval/models.py index cbe08be..cc0972b 100644 --- a/scripts/native_eval/models.py +++ b/scripts/native_eval/models.py @@ -99,7 +99,7 @@ def to_dict(self) -> dict[str, object]: NODE_VERSION = "22.23.1" LITELLM_VERSION = "1.93.0" -REAL_TRAJECTORY_HARNESSES = frozenset({"openclaw", "hermes", "codex"}) +REAL_TRAJECTORY_HARNESSES = frozenset({"openclaw", "hermes", "codex", "claude-code"}) def model_by_slug(slug: str) -> ModelSpec: diff --git a/tests/test_native_eval_runner.py b/tests/test_native_eval_runner.py index 6aef78f..900f0f7 100644 --- a/tests/test_native_eval_runner.py +++ b/tests/test_native_eval_runner.py @@ -29,6 +29,7 @@ MODELS, RunSpec, build_matrix_plan, + trajectory_mode_for_harness, ) from scripts.native_eval import plan as native_plan from scripts.native_eval.proxy import JUDGE_PROXY_MODEL_NAME, write_proxy_config @@ -59,6 +60,17 @@ def test_matrix_plan_contains_only_requested_models_and_harnesses() -> None: assert {run.repetition for run in plan} == {1, 2, 3} +def test_all_native_harnesses_emit_real_trajectories() -> None: + assert { + harness.name: trajectory_mode_for_harness(harness.name) for harness in HARNESSES + } == { + "openclaw": "real_harness_events", + "hermes": "real_harness_events", + "codex": "real_harness_events", + "claude-code": "real_harness_events", + } + + def test_run_index_records_agent_and_judge_reasoning( tmp_path: Path, monkeypatch,