diff --git a/scripts/native_eval/research_audit.py b/scripts/native_eval/research_audit.py index 381df62..63e4317 100644 --- a/scripts/native_eval/research_audit.py +++ b/scripts/native_eval/research_audit.py @@ -1,4 +1,4 @@ -"""Export task, turn, tool, usage, and model-identity research tables.""" +"""Export task, turn, tool, discovery, usage, and identity research tables.""" from __future__ import annotations @@ -33,9 +33,13 @@ "trajectory_path", "toolchain_manifest_path", "proxy_log_path", + "runner_commit", "trajectory_status", + "trace_fidelity", "observed_model_ids", "model_identity_status", + "discovery_status", + "discovery_event_count", "turn_count", "tool_call_count", "n_input_tokens", @@ -85,6 +89,29 @@ "trajectory_path", ) +DISCOVERY_FIELDS = ( + "run_label", + "harness", + "model_slug", + "expected_model_id", + "reasoning_effort", + "repetition", + "task_name", + "turn_index", + "tool_call_id", + "counter_scope", + "operation", + "count", + "count_semantics", + "query", + "selected_id", + "catalog_size", + "source_breakdown_json", + "success", + "trace_fidelity", + "trajectory_path", +) + RUN_AUDIT_FIELDS = ( "run_label", "harness", @@ -303,6 +330,260 @@ def _observation(step: dict[str, Any], call_id: str) -> str: return "\n".join(str(item.get("content") or "") for item in selected) +def _json_values(text: str) -> list[Any]: + stripped = text.strip() + if not stripped: + return [] + try: + return [json.loads(stripped)] + except json.JSONDecodeError: + pass + values: list[Any] = [] + decoder = json.JSONDecoder() + for index, char in enumerate(text): + if char not in "[{": + continue + try: + value, _ = decoder.raw_decode(text[index:]) + except json.JSONDecodeError: + continue + values.append(value) + return values + + +def _find_tool_search_telemetry(value: Any) -> dict[str, Any] | None: + if isinstance(value, dict): + telemetry = value.get("telemetry") + if isinstance(telemetry, dict) and any( + key in telemetry for key in ("catalogSize", "searchCount", "describeCount", "callCount") + ): + return telemetry + for child in value.values(): + found = _find_tool_search_telemetry(child) + if found is not None: + return found + elif isinstance(value, list): + for child in value: + found = _find_tool_search_telemetry(child) + if found is not None: + return found + return None + + +def _openclaw_code_discovery_rows( + *, + base: dict[str, Any], + observation: str, +) -> tuple[list[dict[str, Any]], bool]: + telemetry = next( + ( + found + for value in _json_values(observation) + if (found := _find_tool_search_telemetry(value)) is not None + ), + None, + ) + if telemetry is None: + return [], False + sources = telemetry.get("sources") + source_breakdown = sources if isinstance(sources, dict) else {} + scope_value = telemetry.get("counterScope") + counter_scope = scope_value.strip() if isinstance(scope_value, str) else "" + rows: list[dict[str, Any]] = [] + for operation, key in ( + ("search", "searchCount"), + ("describe", "describeCount"), + ("call", "callCount"), + ): + count = _number(telemetry.get(key)) + if count is None or (count == 0 and not counter_scope): + continue + count_semantics = ( + "invalid_counter_scope" + if count < 0 and not counter_scope + else ( + "cumulative_scoped" + if counter_scope + else "cumulative_unscoped" + ) + ) + rows.append( + { + **base, + "counter_scope": counter_scope, + "operation": operation, + "count": count, + "count_semantics": count_semantics, + "query": "", + "selected_id": "", + "catalog_size": _number(telemetry.get("catalogSize")), + "source_breakdown_json": json.dumps( + source_breakdown, + ensure_ascii=True, + sort_keys=True, + ), + "success": "", + } + ) + return rows, True + + +def _structured_discovery_row( + *, + base: dict[str, Any], + function_name: str, + arguments: Any, +) -> dict[str, Any] | None: + operation = { + "tool_search": "search", + "tool_describe": "describe", + "tool_call": "call", + }.get(function_name) + if operation is None: + return None + values = arguments if isinstance(arguments, dict) else {} + return { + **base, + "counter_scope": "", + "operation": operation, + "count": 1, + "count_semantics": "event", + "query": values.get("query") if operation == "search" else "", + "selected_id": ( + values.get("id") or values.get("toolId") or values.get("name") + if operation in {"describe", "call"} + else "" + ), + "catalog_size": "", + "source_breakdown_json": "{}", + "success": "", + } + + +def _discovery_status( + *, + harness: str, + openclaw_mode: str, + tool_names: list[str], + discovery_rows: list[dict[str, Any]], + telemetry_observed: bool, +) -> str: + if harness == "openclaw": + if any( + row.get("count_semantics") == "invalid_counter_scope" + for row in discovery_rows + ): + return "invalid_counter_scope" + if any( + row.get("count_semantics") == "cumulative_unscoped" + for row in discovery_rows + ): + return "observed_cumulative_unscoped" + if discovery_rows and all( + row.get("count_semantics") == "scope_marker" + for row in discovery_rows + ): + return "supported_not_exercised" + if discovery_rows: + return "observed" + if telemetry_observed: + return "supported_not_exercised" + if any( + name + in { + "tool_search_code", + "tool_search", + "tool_describe", + "tool_call", + } + for name in tool_names + ): + return "unobservable" + return "supported_not_exercised" if openclaw_mode else "disabled" + if harness == "codex": + return "unobservable_native_stream" + return "unsupported" + + +def _discovery_operation_count(rows: list[dict[str, Any]]) -> int: + return sum( + int(_number(row.get("count")) or 0) + for row in rows + if row.get("count_semantics") + not in { + "cumulative_scoped", + "cumulative_unscoped", + "invalid_counter_scope", + } + ) + + +def _normalize_cumulative_discovery_rows( + rows: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Convert scoped cumulative telemetry into deltas without guessing resets.""" + previous_counts: dict[tuple[str, str], int] = {} + normalized_rows: dict[tuple[str, str], list[dict[str, Any]]] = {} + invalid_keys: set[tuple[str, str]] = set() + normalized: list[dict[str, Any]] = [] + for row in rows: + semantics = row.get("count_semantics") + if semantics != "cumulative_scoped": + normalized.append(row) + continue + scope = str(row.get("counter_scope") or "") + operation = str(row.get("operation") or "") + count = int(_number(row.get("count")) or 0) + key = (scope, operation) + if key in invalid_keys: + normalized.append({**row, "count_semantics": "invalid_counter_scope"}) + continue + delta = count - previous_counts.get(key, 0) + previous_counts[key] = count + if delta < 0: + invalid_keys.add(key) + for prior_row in normalized_rows.get(key, []): + prior_row["count_semantics"] = "invalid_counter_scope" + invalid_row = {**row, "count_semantics": "invalid_counter_scope"} + normalized.append(invalid_row) + elif delta > 0: + delta_row = {**row, "count": delta, "count_semantics": "delta"} + normalized_rows.setdefault(key, []).append(delta_row) + normalized.append(delta_row) + else: + marker_row = { + **row, + "count": 0, + "count_semantics": "scope_marker", + } + normalized_rows.setdefault(key, []).append(marker_row) + normalized.append(marker_row) + return normalized + + +def _invalidate_cross_task_counter_scopes( + rows: list[dict[str, Any]], +) -> None: + owners_by_scope: dict[tuple[str, str], set[tuple[str, str]]] = {} + rows_by_scope: dict[tuple[str, str], list[dict[str, Any]]] = {} + for row in rows: + scope = str(row.get("counter_scope") or "") + if not scope: + continue + scope_key = (str(row.get("run_label") or ""), scope) + owner = ( + str(row.get("task_name") or ""), + str(row.get("trajectory_path") or ""), + ) + owners_by_scope.setdefault(scope_key, set()).add(owner) + rows_by_scope.setdefault(scope_key, []).append(row) + for scope_key, owners in owners_by_scope.items(): + if len(owners) < 2: + continue + for row in rows_by_scope[scope_key]: + row["count_semantics"] = "invalid_counter_scope" + + def _write_csv(path: Path, fields: tuple[str, ...], rows: list[dict[str, Any]]) -> None: with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore") @@ -382,6 +663,7 @@ def export_research_tables( trace_rows: list[dict[str, Any]] = [] turn_rows: list[dict[str, Any]] = [] tool_rows: list[dict[str, Any]] = [] + discovery_rows: list[dict[str, Any]] = [] run_rows: list[dict[str, Any]] = [] for entry_value in run_index["runs"]: @@ -400,6 +682,14 @@ def export_research_tables( requested_harness_version, installed_harness_version, ) + job_manifest = _read_json(job_dir / "run_manifest.json") if job_dir else {} + job_manifest = job_manifest or {} + runner_commit = str(job_manifest.get("runner_commit") or "") + openclaw_mode = str( + entry.get("openclaw_tool_search_mode") + or job_manifest.get("openclaw_tool_search_mode") + or "" + ) proxy_log_path = proxy_logs.get(run_label) counters: Counter[str] = Counter() if job_dir is not None: @@ -434,6 +724,7 @@ def export_research_tables( steps = trajectory.get("steps") if not isinstance(steps, list): steps = [] + trace_fidelity = str(_nested(trajectory, "extra", "trace_fidelity") or "") input_tokens, cache_tokens, output_tokens = _usage(agent_result, trajectory) cost, cost_provenance = _cost(agent_result, trajectory) counters[ @@ -442,6 +733,9 @@ def export_research_tables( else "task_cost_unavailable" ] += 1 task_tool_count = 0 + task_tool_names: list[str] = [] + task_discovery_rows: list[dict[str, Any]] = [] + task_discovery_telemetry_observed = False for turn_index, step_value in enumerate(steps): if not isinstance(step_value, dict): @@ -481,6 +775,8 @@ def export_research_tables( call_id = str(call_value.get("tool_call_id") or "") observation = _observation(step, call_id) arguments = call_value.get("arguments") + function_name = str(call_value.get("function_name") or "") + task_tool_names.append(function_name) tool_rows.append( { "run_label": run_label, @@ -493,7 +789,7 @@ def export_research_tables( "turn_index": turn_index, "tool_index": tool_index, "tool_call_id": call_id, - "function_name": call_value.get("function_name"), + "function_name": function_name, "arguments_json": json.dumps( arguments, ensure_ascii=True, @@ -504,6 +800,50 @@ def export_research_tables( "trajectory_path": str(trajectory_path), } ) + discovery_base = { + "run_label": run_label, + "harness": harness, + "model_slug": entry.get("model_slug"), + "expected_model_id": expected_model_id, + "reasoning_effort": entry.get("reasoning_effort"), + "repetition": entry.get("repetition"), + "task_name": task_name, + "turn_index": turn_index, + "tool_call_id": call_id, + "trace_fidelity": trace_fidelity, + "trajectory_path": str(trajectory_path), + } + if harness == "openclaw" and function_name == "tool_search_code": + code_rows, telemetry_observed = _openclaw_code_discovery_rows( + base=discovery_base, + observation=observation, + ) + task_discovery_rows.extend(code_rows) + task_discovery_telemetry_observed |= telemetry_observed + elif harness == "openclaw": + discovery_row = _structured_discovery_row( + base=discovery_base, + function_name=function_name, + arguments=arguments, + ) + if discovery_row is not None: + task_discovery_rows.append(discovery_row) + + if harness == "openclaw" and openclaw_mode == "code": + task_discovery_rows = _normalize_cumulative_discovery_rows( + task_discovery_rows + ) + discovery_status = _discovery_status( + harness=harness, + openclaw_mode=openclaw_mode, + tool_names=task_tool_names, + discovery_rows=task_discovery_rows, + telemetry_observed=task_discovery_telemetry_observed, + ) + task_discovery_operation_count = _discovery_operation_count( + task_discovery_rows + ) + discovery_rows.extend(task_discovery_rows) trace_rows.append( { @@ -532,9 +872,13 @@ def export_research_tables( str(toolchain_path) if toolchain_path else "" ), "proxy_log_path": str(proxy_log_path) if proxy_log_path else "", + "runner_commit": runner_commit, "trajectory_status": agent_result.get("trajectory_status"), + "trace_fidelity": trace_fidelity, "observed_model_ids": json.dumps(sorted(observed)), "model_identity_status": identity_status, + "discovery_status": discovery_status, + "discovery_event_count": task_discovery_operation_count, "turn_count": len(steps), "tool_call_count": task_tool_count, "n_input_tokens": input_tokens, @@ -586,15 +930,46 @@ def export_research_tables( } ) + _invalidate_cross_task_counter_scopes(discovery_rows) + discovery_rows_by_trace: dict[ + tuple[str, str, str], list[dict[str, Any]] + ] = {} + for row in discovery_rows: + key = ( + str(row.get("run_label") or ""), + str(row.get("task_name") or ""), + str(row.get("trajectory_path") or ""), + ) + discovery_rows_by_trace.setdefault(key, []).append(row) + for trace_row in trace_rows: + key = ( + str(trace_row.get("run_label") or ""), + str(trace_row.get("task_name") or ""), + str(trace_row.get("trajectory_path") or ""), + ) + task_rows = discovery_rows_by_trace.get(key, []) + if not task_rows: + continue + trace_row["discovery_status"] = _discovery_status( + harness=str(trace_row.get("harness") or ""), + openclaw_mode=str(trace_row.get("openclaw_tool_search_mode") or ""), + tool_names=[], + discovery_rows=task_rows, + telemetry_observed=True, + ) + trace_row["discovery_event_count"] = _discovery_operation_count(task_rows) + _write_csv(output_dir / "trace_inventory.csv", TRACE_FIELDS, trace_rows) _write_csv(output_dir / "turn_usage.csv", TURN_FIELDS, turn_rows) _write_csv(output_dir / "tool_calls.csv", TOOL_FIELDS, tool_rows) + _write_csv(output_dir / "discovery_events.csv", DISCOVERY_FIELDS, discovery_rows) _write_csv(output_dir / "model_identity_audit.csv", RUN_AUDIT_FIELDS, run_rows) summary = { "run_count": len(run_rows), "task_result_count": len(trace_rows), "turn_count": len(turn_rows), "tool_call_count": len(tool_rows), + "discovery_event_count": _discovery_operation_count(discovery_rows), "identity_audit_pass_count": sum( row["model_identity_audit_passed"] is True for row in run_rows ), @@ -616,6 +991,7 @@ def export_research_tables( "trace_inventory": "trace_inventory.csv", "turn_usage": "turn_usage.csv", "tool_calls": "tool_calls.csv", + "discovery_events": "discovery_events.csv", "model_identity_audit": "model_identity_audit.csv", }, } diff --git a/tests/test_native_eval_research_audit.py b/tests/test_native_eval_research_audit.py index 8e37557..4000521 100644 --- a/tests/test_native_eval_research_audit.py +++ b/tests/test_native_eval_research_audit.py @@ -4,7 +4,15 @@ import json from pathlib import Path -from scripts.native_eval.research_audit import export_research_tables +from scripts.native_eval.research_audit import ( + _discovery_operation_count, + _discovery_status, + _invalidate_cross_task_counter_scopes, + _normalize_cumulative_discovery_rows, + _openclaw_code_discovery_rows, + _structured_discovery_row, + export_research_tables, +) def _write_json(path: Path, value: object) -> None: @@ -12,6 +20,298 @@ def _write_json(path: Path, value: object) -> None: path.write_text(json.dumps(value), encoding="utf-8") +def test_structured_discovery_success_stays_unknown_without_result_status() -> None: + row = _structured_discovery_row( + base={}, + function_name="tool_call", + arguments={"id": "openclaw:core:read"}, + ) + + assert row is not None + assert row["success"] == "" + + +def test_zero_count_code_telemetry_is_observed_but_not_exercised() -> None: + rows, telemetry_observed = _openclaw_code_discovery_rows( + base={}, + observation=json.dumps( + { + "telemetry": { + "catalogSize": 12, + "counterScope": "scope-zero", + "searchCount": 0, + "describeCount": 0, + "callCount": 0, + } + } + ), + ) + + normalized = _normalize_cumulative_discovery_rows(rows) + assert len(normalized) == 3 + assert {row["count_semantics"] for row in normalized} == {"scope_marker"} + assert telemetry_observed is True + assert ( + _discovery_status( + harness="openclaw", + openclaw_mode="code", + tool_names=["tool_search_code"], + discovery_rows=normalized, + telemetry_observed=telemetry_observed, + ) + == "supported_not_exercised" + ) + + +def test_zero_count_unscoped_telemetry_is_not_exercised() -> None: + rows, telemetry_observed = _openclaw_code_discovery_rows( + base={}, + observation=json.dumps( + { + "telemetry": { + "catalogSize": 12, + "searchCount": 0, + "describeCount": 0, + "callCount": 0, + } + } + ), + ) + + assert rows == [] + assert ( + _discovery_status( + harness="openclaw", + openclaw_mode="code", + tool_names=["tool_search_code"], + discovery_rows=rows, + telemetry_observed=telemetry_observed, + ) + == "supported_not_exercised" + ) + + +def test_cumulative_code_telemetry_exports_deltas_within_each_scope() -> None: + rows = [ + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "search", + "count": 1, + "tool_call_id": "call-1", + }, + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "call", + "count": 2, + "tool_call_id": "call-1", + }, + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "search", + "count": 1, + "tool_call_id": "call-2", + }, + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "call", + "count": 5, + "tool_call_id": "call-2", + }, + { + "counter_scope": "scope-b", + "count_semantics": "cumulative_scoped", + "operation": "search", + "count": 2, + "tool_call_id": "call-3", + }, + { + "counter_scope": "scope-b", + "count_semantics": "cumulative_scoped", + "operation": "call", + "count": 6, + "tool_call_id": "call-3", + }, + ] + + normalized = _normalize_cumulative_discovery_rows(rows) + + assert [ + ( + row["tool_call_id"], + row["operation"], + row["count"], + row["count_semantics"], + ) + for row in normalized + ] == [ + ("call-1", "search", 1, "delta"), + ("call-1", "call", 2, "delta"), + ("call-2", "search", 0, "scope_marker"), + ("call-2", "call", 3, "delta"), + ("call-3", "search", 2, "delta"), + ("call-3", "call", 6, "delta"), + ] + assert _discovery_operation_count(normalized) == 14 + + +def test_counter_scope_reused_across_tasks_is_invalid() -> None: + rows = [ + { + "run_label": "run-a", + "task_name": task_name, + "trajectory_path": trajectory_path, + "counter_scope": "scope-a", + "count_semantics": "delta", + "operation": "call", + "count": count, + } + for task_name, trajectory_path, count in ( + ("task-a", "/traces/a.json", 5), + ("task-b", "/traces/b.json", 2), + ) + ] + + _invalidate_cross_task_counter_scopes(rows) + + assert {row["count_semantics"] for row in rows} == { + "invalid_counter_scope" + } + assert _discovery_operation_count(rows) == 0 + + +def test_zero_scope_marker_participates_in_cross_task_validation() -> None: + rows = [ + { + "run_label": "run-a", + "task_name": "task-a", + "trajectory_path": "/traces/a.json", + "counter_scope": "scope-a", + "count_semantics": "scope_marker", + "operation": "call", + "count": 0, + }, + { + "run_label": "run-a", + "task_name": "task-b", + "trajectory_path": "/traces/b.json", + "counter_scope": "scope-a", + "count_semantics": "delta", + "operation": "call", + "count": 2, + }, + ] + + _invalidate_cross_task_counter_scopes(rows) + + assert {row["count_semantics"] for row in rows} == { + "invalid_counter_scope" + } + + +def test_unscoped_cumulative_telemetry_stays_visible_but_unresolved() -> None: + rows, telemetry_observed = _openclaw_code_discovery_rows( + base={}, + observation=json.dumps( + { + "telemetry": { + "catalogSize": 12, + "searchCount": 3, + "describeCount": 1, + "callCount": 8, + } + } + ), + ) + + normalized = _normalize_cumulative_discovery_rows(rows) + + assert {row["count_semantics"] for row in normalized} == { + "cumulative_unscoped" + } + assert _discovery_operation_count(normalized) == 0 + assert ( + _discovery_status( + harness="openclaw", + openclaw_mode="code", + tool_names=["tool_search_code"], + discovery_rows=normalized, + telemetry_observed=telemetry_observed, + ) + == "observed_cumulative_unscoped" + ) + + +def test_invalid_counter_takes_precedence_over_unscoped_telemetry() -> None: + rows, telemetry_observed = _openclaw_code_discovery_rows( + base={}, + observation=json.dumps( + { + "telemetry": { + "catalogSize": 12, + "searchCount": -1, + "describeCount": 2, + "callCount": 0, + } + } + ), + ) + + assert ( + _discovery_status( + harness="openclaw", + openclaw_mode="code", + tool_names=["tool_search_code"], + discovery_rows=rows, + telemetry_observed=telemetry_observed, + ) + == "invalid_counter_scope" + ) + + +def test_counter_regression_quarantines_that_scope_operation() -> None: + rows = [ + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "call", + "count": count, + "tool_call_id": f"call-{index}", + } + for index, count in enumerate((5, 5, 3, 4), start=1) + ] + + normalized = _normalize_cumulative_discovery_rows(rows) + + assert len(normalized) == 4 + assert {row["count_semantics"] for row in normalized} == { + "invalid_counter_scope" + } + assert _discovery_operation_count(normalized) == 0 + + +def test_negative_scoped_counter_quarantines_prior_deltas() -> None: + rows = [ + { + "counter_scope": "scope-a", + "count_semantics": "cumulative_scoped", + "operation": "call", + "count": count, + } + for count in (5, -1) + ] + + normalized = _normalize_cumulative_discovery_rows(rows) + + assert {row["count_semantics"] for row in normalized} == { + "invalid_counter_scope" + } + assert _discovery_operation_count(normalized) == 0 + + def test_research_audit_exports_identity_turn_tool_and_usage_tables( tmp_path: Path, ) -> None: @@ -132,6 +432,7 @@ def test_research_audit_exports_identity_turn_tool_and_usage_tables( ) assert trace_row["phase"] == "full" assert trace_row["cost_provenance"] == "exact_harness" + assert trace_row["discovery_status"] == "disabled" with (output / "turn_usage.csv").open(newline="", encoding="utf-8") as handle: turn_row = next(csv.DictReader(handle)) assert turn_row["n_input_tokens"] == "12" @@ -140,6 +441,141 @@ def test_research_audit_exports_identity_turn_tool_and_usage_tables( tool_row = next(csv.DictReader(handle)) assert tool_row["function_name"] == "shell" assert tool_row["observation_excerpt"] == "/workspace" + with (output / "discovery_events.csv").open( + newline="", + encoding="utf-8", + ) as handle: + assert list(csv.DictReader(handle)) == [] + + +def test_research_audit_exports_openclaw_code_discovery_telemetry( + tmp_path: Path, +) -> None: + run_label = "openclaw-gpt55-high-ablation-1-r1-20260729" + run_index = tmp_path / "run-index.json" + extracted = tmp_path / "extracted" + output = tmp_path / "analysis" + job_dir = extracted / run_label + trial_dir = job_dir / "task__abc" + trajectory_path = trial_dir / "agent" / "trajectory.json" + _write_json( + run_index, + { + "runs": [ + { + "run_label": run_label, + "harness": "openclaw", + "harness_version": "2026.7.1-2", + "model_slug": "gpt55", + "model_id": "gpt-5.5", + "reasoning_effort": "high", + "repetition": 1, + "expected_task_count": 1, + "openclaw_tool_search_mode": "code", + } + ] + }, + ) + _write_json( + job_dir / "run_manifest.json", + { + "run_label": run_label, + "runner_commit": "runner-sha", + "openclaw_tool_search_mode": "code", + }, + ) + _write_json( + trial_dir / "result.json", + { + "task_id": {"path": "/tasks/example-task"}, + "agent_result": { + "trajectory_status": "real", + "runtime_model_name": "gpt-5.5", + "canonical_model_identity": True, + }, + }, + ) + _write_json( + trajectory_path, + { + "agent": { + "name": "openclaw", + "version": "2026.7.1-2", + "model_name": "openai/gpt-5.5", + }, + "steps": [ + { + "source": "agent", + "tool_calls": [ + { + "tool_call_id": "call-1", + "function_name": "tool_search_code", + "arguments": { + "code": "return await openclaw.tools.search('mail')" + }, + } + ], + "observation": { + "results": [ + { + "source_call_id": "call-1", + "content": json.dumps( + { + "ok": True, + "telemetry": { + "catalogSize": 42, + "counterScope": "scope-a", + "sources": { + "openclaw": 30, + "mcp": 10, + "client": 2, + }, + "searchCount": 2, + "describeCount": 1, + "callCount": 1, + }, + } + ), + } + ] + }, + } + ], + "extra": { + "observed_models": ["gpt-5.5"], + "trace_fidelity": "session", + }, + }, + ) + + summary = export_research_tables( + run_index_path=run_index, + extracted_root=extracted, + output_dir=output, + ) + + assert summary["discovery_event_count"] == 4 + with (output / "trace_inventory.csv").open(newline="", encoding="utf-8") as handle: + trace_row = next(csv.DictReader(handle)) + assert trace_row["runner_commit"] == "runner-sha" + assert trace_row["trace_fidelity"] == "session" + assert trace_row["discovery_status"] == "observed" + assert trace_row["discovery_event_count"] == "4" + with (output / "discovery_events.csv").open( + newline="", + encoding="utf-8", + ) as handle: + rows = list(csv.DictReader(handle)) + assert [(row["operation"], row["count"]) for row in rows] == [ + ("search", "2"), + ("describe", "1"), + ("call", "1"), + ] + assert {row["counter_scope"] for row in rows} == {"scope-a"} + assert {row["count_semantics"] for row in rows} == {"delta"} + assert {row["catalog_size"] for row in rows} == {"42"} + assert {row["success"] for row in rows} == {""} + assert {row["trace_fidelity"] for row in rows} == {"session"} def test_research_audit_fails_identity_when_trace_is_missing(tmp_path: Path) -> None: