From 2b49af7cc018eac39653b7f797b15f521a3501c3 Mon Sep 17 00:00:00 2001 From: totoleon Date: Mon, 13 Jul 2026 22:18:51 +0000 Subject: [PATCH] Add field to track time to first response and token usages --- evalbench/eval_service.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/evalbench/eval_service.py b/evalbench/eval_service.py index 70b81084..0976b7e9 100644 --- a/evalbench/eval_service.py +++ b/evalbench/eval_service.py @@ -1,8 +1,10 @@ """A gRPC servicer that handles EvalService requests.""" +import ast import asyncio import json import os +import pandas as pd from collections.abc import AsyncIterator from typing import AsyncGenerator @@ -514,4 +516,31 @@ def _process_results( "p90": round(latencies.quantile(0.9), 2), } + # Add percentiles for metrics reported via the `other` map. `other` is + # stored as a string-serialized dict in the dataframe, so parse it first. + if "other" in results_df.columns: + def _parse_other(o): + if isinstance(o, dict): + return o + if isinstance(o, str) and o: + try: + return ast.literal_eval(o) + except (ValueError, SyntaxError): + return {} + return {} + + parsed_other = results_df["other"].apply(_parse_other) + for metric_key in ( + "time_to_first_response", + "input_token_count", + "output_token_count", + ): + values = parsed_other.apply(lambda d, k=metric_key: d.get(k)) + values = pd.to_numeric(values, errors="coerce").dropna() + if not values.empty: + summary[metric_key] = { + "p50": round(values.quantile(0.5), 2), + "p90": round(values.quantile(0.9), 2), + } + return summary