fix(evaluation): isolate EvalCases per trace in tracing JSON import (#1021)

tests/test_evaluator.py

@@ -122,6 +122,87 @@
 ]
 
 
+MULTI_TRACE_SET_DATA = [
+    # Trace A (app_a / user_a). Spans are deliberately NOT ordered by
+    # start_time: the second call_llm span comes first in the file to
+    # exercise the start_time sorting fix (issue #1021).
+    {
+        "name": "call_llm",
+        "span_id": 2001,
+        "trace_id": 11111111111111111111111111111111,
+        "start_time": 1758158957171713000,
+        "end_time": 1758158964035230000,
+        "attributes": {
+            "gen_ai.app.name": "app_a",
+            "gen_ai.user.id": "user_a",
+            "gen_ai.prompt.0.content": "follow-up A",
+            "gen_ai.completion.0.content": "response A",
+        },
+        "parent_span_id": 1000,
+    },
+    {
+        "name": "execute_tool get_city_weather",
+        "span_id": 2002,
+        "trace_id": 11111111111111111111111111111111,
+        "start_time": 1758158957162250000,
+        "end_time": 1758158957162426000,
+        "attributes": {
+            "gen_ai.tool.name": "get_city_weather",
+            "gen_ai.tool.input": '{"name": "get_city_weather", "parameters": {"city": "Beijing"}}',
+            "gen_ai.tool.output": '{"id": "call_w4bj25flpvs74zgyyiquqh5s", "name": "get_city_weather", "response": {"result": "Sunny, 25°C"}}',
+        },
+        "parent_span_id": 1000,
+    },
+    {
+        "name": "call_llm",
+        "span_id": 1001,
+        "trace_id": 11111111111111111111111111111111,
+        "start_time": 1758158945807630000,
+        "end_time": 1758158957171304000,
+        "attributes": {
+            "gen_ai.app.name": "app_a",
+            "gen_ai.user.id": "user_a",
+            "gen_ai.prompt.0.role": "user",
+            "gen_ai.prompt.0.content": "hello A",
+        },
+        "parent_span_id": 1000,
+    },
+    {
+        "name": "invocation",
+        "span_id": 1000,
+        "trace_id": 11111111111111111111111111111111,
+        "start_time": 1758158945807233000,
+        "end_time": 1758158964035304000,
+        "attributes": {},
+        "parent_span_id": None,
+    },
+    # Trace B (app_b / user_b): single call_llm span, no tool uses.
+    {
+        "name": "call_llm",
+        "span_id": 3001,
+        "trace_id": 22222222222222222222222222222222,
+        "start_time": 1758159045807630000,
+        "end_time": 1758159047171304000,
+        "attributes": {
+            "gen_ai.app.name": "app_b",
+            "gen_ai.user.id": "user_b",
+            "gen_ai.prompt.0.content": "hello B",
+            "gen_ai.completion.0.content": "response B",
+        },
+        "parent_span_id": 3000,
+    },
+    {
+        "name": "invocation",
+        "span_id": 3000,
+        "trace_id": 22222222222222222222222222222222,
+        "start_time": 1758159045807233000,
+        "end_time": 1758159047171304000,
+        "attributes": {},
+        "parent_span_id": None,
+    },
+]
+
+
 def test_evaluator():
     base_evaluator = BaseEvaluator(agent=None, name="test_evaluator")
 
@@ -160,3 +241,46 @@ def test_tracing_file_to_evalset():
     )
 
     os.remove(tracing_file_path)
+
+
+def test_tracing_file_multiple_traces_to_evalset():
+    """Regression test for #1021: multi-trace files must not collapse into one
+    eval case.
+
+    Each trace_id must produce one isolated EvalCase, spans must be ordered by
+    start_time within a trace, and conversation/tool uses/session metadata must
+    never cross trace boundaries.
+    """
+    base_evaluator = BaseEvaluator(agent=None, name="test_evaluator")
+
+    tracing_file_path = "./tracing_for_test_evaluator_multiple_traces.json"
+    with open(tracing_file_path, "w") as f:
+        json.dump(MULTI_TRACE_SET_DATA, f)
+
+    base_evaluator.build_eval_set(file_path=tracing_file_path)
+
+    # Two traces -> two isolated eval cases
+    assert len(base_evaluator.invocation_list) == 2
+    assert len(base_evaluator.agent_information_list) == 2
+
+    # First case: trace A metadata and conversation, spans were out of order
+    first_case = base_evaluator.invocation_list[0]
+    assert base_evaluator.agent_information_list[0]["app_name"] == "app_a"
+    assert base_evaluator.agent_information_list[0]["user_id"] == "user_a"
+    assert len(first_case.invocations) == 1
+    assert first_case.invocations[0].input == "hello A"
+    assert first_case.invocations[0].expected_output == "response A"
+    assert first_case.invocations[0].expected_tool == [
+        {"name": "get_city_weather", "args": {"city": "Beijing"}}
+    ]
+
+    # Second case: trace B metadata and conversation
+    second_case = base_evaluator.invocation_list[1]
+    assert base_evaluator.agent_information_list[1]["app_name"] == "app_b"
+    assert base_evaluator.agent_information_list[1]["user_id"] == "user_b"
+    assert len(second_case.invocations) == 1
+    assert second_case.invocations[0].input == "hello B"
+    assert second_case.invocations[0].expected_output == "response B"
+    assert second_case.invocations[0].expected_tool == []
+
+    os.remove(tracing_file_path)

veadk/evaluation/base_evaluator.py

@@ -262,20 +262,26 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
         except Exception as e:
             raise ValueError(f"Error reading file {tracing_json_path}: {e}")
 
-        # Group spans by trace_id
+        # Group spans by trace_id and sort each trace's spans by start_time so
+        # the first/last `call_llm` spans reliably map to the user input and
+        # final output within that trace.
         trace_groups = {}
         for span in tracing_data:
             trace_id = span["trace_id"]
             if trace_id not in trace_groups:
                 trace_groups[trace_id] = []
             trace_groups[trace_id].append(span)
 
-        # Convert to evalset format
-        eval_cases, conversation = [], []
-        app_name, user_id = "", ""
-        creation_timestamp = 0
+        # Convert to evalset format. Each trace_id becomes one isolated
+        # EvalCase so conversation, tool uses, and session metadata never cross
+        # trace boundaries.
+        eval_cases = []
         for trace_id, spans in trace_groups.items():
+            spans = sorted(spans, key=lambda span: span.get("start_time", 0))
             tool_uses = []
+            conversation = []
+            app_name, user_id = "", ""
+            creation_timestamp = 0
 
             # Extract tool_uses from spans with name starting with "execute_tool"
             for span in spans:
@@ -347,17 +353,22 @@ def _build_eval_set_from_tracing_json(self, tracing_json_path: str) -> EvalSet:
                     }
                 )
 
-        eval_cases.append(
-            {
-                "eval_id": f"veadk_eval_{formatted_timestamp()}",
-                "conversation": conversation,
-                "session_input": {
-                    "app_name": app_name,
-                    "user_id": user_id,
-                    "state": {},
-                },
-                "creation_timestamp": creation_timestamp,
-            }
+            eval_cases.append(
+                {
+                    "eval_id": f"veadk_eval_{formatted_timestamp()}",
+                    "conversation": conversation,
+                    "session_input": {
+                        "app_name": app_name,
+                        "user_id": user_id,
+                        "state": {},
+                    },
+                    "creation_timestamp": creation_timestamp,
+                }
+            )
+
+        # The EvalSet timestamp is the earliest generated case timestamp.
+        creation_timestamp = min(
+            (case["creation_timestamp"] for case in eval_cases), default=0
         )
 
         evalset = EvalSet(