From 52a2ca137ddab03b9a9110b81fdcd118b71ff9f0 Mon Sep 17 00:00:00 2001 From: klioen Date: Mon, 7 Sep 2026 13:57:09 +0000 Subject: [PATCH] fix(evaluation): isolate EvalCases per trace in tracing JSON import When building an EvalSet from a tracing JSON file containing multiple traces, the previous code accumulated all spans into a single conversation and a single EvalCase, mixing tool uses, session metadata (app_name/user_id) and user/model turns across independent sessions. Fix _build_eval_set_from_tracing_json to: - group spans by trace_id and sort each trace's spans by start_time so the first/last call_llm spans map to that trace's input/output - emit one isolated EvalCase per trace - derive the EvalSet creation timestamp from the earliest case Adds a regression test with two traces (one with out-of-order spans) verifying cases, metadata, tools and conversations stay isolated. Fixes #1021 --- tests/test_evaluator.py | 124 +++++++++++++++++++++++++++++ veadk/evaluation/base_evaluator.py | 43 ++++++---- 2 files changed, 151 insertions(+), 16 deletions(-) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 05ced6be6..35ee0e2b1 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -122,6 +122,87 @@ ] +MULTI_TRACE_SET_DATA = [ + # Trace A (app_a / user_a). Spans are deliberately NOT ordered by + # start_time: the second call_llm span comes first in the file to + # exercise the start_time sorting fix (issue #1021). + { + "name": "call_llm", + "span_id": 2001, + "trace_id": 11111111111111111111111111111111, + "start_time": 1758158957171713000, + "end_time": 1758158964035230000, + "attributes": { + "gen_ai.app.name": "app_a", + "gen_ai.user.id": "user_a", + "gen_ai.prompt.0.content": "follow-up A", + "gen_ai.completion.0.content": "response A", + }, + "parent_span_id": 1000, + }, + { + "name": "execute_tool get_city_weather", + "span_id": 2002, + "trace_id": 11111111111111111111111111111111, + "start_time": 1758158957162250000, + "end_time": 1758158957162426000, + "attributes": { + "gen_ai.tool.name": "get_city_weather", + "gen_ai.tool.input": '{"name": "get_city_weather", "parameters": {"city": "Beijing"}}', + "gen_ai.tool.output": '{"id": "call_w4bj25flpvs74zgyyiquqh5s", "name": "get_city_weather", "response": {"result": "Sunny, 25°C"}}', + }, + "parent_span_id": 1000, + }, + { + "name": "call_llm", + "span_id": 1001, + "trace_id": 11111111111111111111111111111111, + "start_time": 1758158945807630000, + "end_time": 1758158957171304000, + "attributes": { + "gen_ai.app.name": "app_a", + "gen_ai.user.id": "user_a", + "gen_ai.prompt.0.role": "user", + "gen_ai.prompt.0.content": "hello A", + }, + "parent_span_id": 1000, + }, + { + "name": "invocation", + "span_id": 1000, + "trace_id": 11111111111111111111111111111111, + "start_time": 1758158945807233000, + "end_time": 1758158964035304000, + "attributes": {}, + "parent_span_id": None, + }, + # Trace B (app_b / user_b): single call_llm span, no tool uses. + { + "name": "call_llm", + "span_id": 3001, + "trace_id": 22222222222222222222222222222222, + "start_time": 1758159045807630000, + "end_time": 1758159047171304000, + "attributes": { + "gen_ai.app.name": "app_b", + "gen_ai.user.id": "user_b", + "gen_ai.prompt.0.content": "hello B", + "gen_ai.completion.0.content": "response B", + }, + "parent_span_id": 3000, + }, + { + "name": "invocation", + "span_id": 3000, + "trace_id": 22222222222222222222222222222222, + "start_time": 1758159045807233000, + "end_time": 1758159047171304000, + "attributes": {}, + "parent_span_id": None, + }, +] + + def test_evaluator(): base_evaluator = BaseEvaluator(agent=None, name="test_evaluator") @@ -160,3 +241,46 @@ def test_tracing_file_to_evalset(): ) os.remove(tracing_file_path) + + +def test_tracing_file_multiple_traces_to_evalset(): + """Regression test for #1021: multi-trace files must not collapse into one + eval case. + + Each trace_id must produce one isolated EvalCase, spans must be ordered by + start_time within a trace, and conversation/tool uses/session metadata must + never cross trace boundaries. + """ + base_evaluator = BaseEvaluator(agent=None, name="test_evaluator") + + tracing_file_path = "./tracing_for_test_evaluator_multiple_traces.json" + with open(tracing_file_path, "w") as f: + json.dump(MULTI_TRACE_SET_DATA, f) + + base_evaluator.build_eval_set(file_path=tracing_file_path) + + # Two traces -> two isolated eval cases + assert len(base_evaluator.invocation_list) == 2 + assert len(base_evaluator.agent_information_list) == 2 + + # First case: trace A metadata and conversation, spans were out of order + first_case = base_evaluator.invocation_list[0] + assert base_evaluator.agent_information_list[0]["app_name"] == "app_a" + assert base_evaluator.agent_information_list[0]["user_id"] == "user_a" + assert len(first_case.invocations) == 1 + assert first_case.invocations[0].input == "hello A" + assert first_case.invocations[0].expected_output == "response A" + assert first_case.invocations[0].expected_tool == [ + {"name": "get_city_weather", "args": {"city": "Beijing"}} + ] + + # Second case: trace B metadata and conversation + second_case = base_evaluator.invocation_list[1] + assert base_evaluator.agent_information_list[1]["app_name"] == "app_b" + assert base_evaluator.agent_information_list[1]["user_id"] == "user_b" + assert len(second_case.invocations) == 1 + assert second_case.invocations[0].input == "hello B" + assert second_case.invocations[0].expected_output == "response B" + assert second_case.invocations[0].expected_tool == [] + + os.remove(tracing_file_path) diff --git a/veadk/evaluation/base_evaluator.py b/veadk/evaluation/base_evaluator.py index 5d872ffdc..e53f88732 100644 --- a/veadk/evaluation/base_evaluator.py +++ b/veadk/evaluation/base_evaluator.py @@ -262,7 +262,9 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet: except Exception as e: raise ValueError(f"Error reading file {tracing_json_path}: {e}") - # Group spans by trace_id + # Group spans by trace_id and sort each trace's spans by start_time so + # the first/last `call_llm` spans reliably map to the user input and + # final output within that trace. trace_groups = {} for span in tracing_data: trace_id = span["trace_id"] @@ -270,12 +272,16 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet: trace_groups[trace_id] = [] trace_groups[trace_id].append(span) - # Convert to evalset format - eval_cases, conversation = [], [] - app_name, user_id = "", "" - creation_timestamp = 0 + # Convert to evalset format. Each trace_id becomes one isolated + # EvalCase so conversation, tool uses, and session metadata never cross + # trace boundaries. + eval_cases = [] for trace_id, spans in trace_groups.items(): + spans = sorted(spans, key=lambda span: span.get("start_time", 0)) tool_uses = [] + conversation = [] + app_name, user_id = "", "" + creation_timestamp = 0 # Extract tool_uses from spans with name starting with "execute_tool" for span in spans: @@ -347,17 +353,22 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet: } ) - eval_cases.append( - { - "eval_id": f"veadk_eval_{formatted_timestamp()}", - "conversation": conversation, - "session_input": { - "app_name": app_name, - "user_id": user_id, - "state": {}, - }, - "creation_timestamp": creation_timestamp, - } + eval_cases.append( + { + "eval_id": f"veadk_eval_{formatted_timestamp()}", + "conversation": conversation, + "session_input": { + "app_name": app_name, + "user_id": user_id, + "state": {}, + }, + "creation_timestamp": creation_timestamp, + } + ) + + # The EvalSet timestamp is the earliest generated case timestamp. + creation_timestamp = min( + (case["creation_timestamp"] for case in eval_cases), default=0 ) evalset = EvalSet(