From 53f4956501f3da15f41ddf2803be08609db4fee6 Mon Sep 17 00:00:00 2001 From: justcodebruh Date: Thu, 14 May 2026 16:53:03 -0400 Subject: [PATCH] fix: handle RemoteEvalParameters in eval dev /list endpoint `build_eval_definitions` crashed with `AttributeError: 'RemoteEvalParameters' object has no attribute 'items'` when an `Eval(...)` passed `parameters=load_parameters(...)`. The runner called `parameters_to_json_schema(evaluator.parameters)` unconditionally; that helper expects an `EvalParameters` mapping and calls `.items()` on it, which `RemoteEvalParameters` does not implement. The dev server's `/list` handler in Rust only surfaced this as a generic 500 ("Eval runner exited with an error."), so the Playground Remote evals settings page just said "could not connect to the dev server" with no indication that the saved-parameters path was the trigger. Branch on `RemoteEvalParameters` and pass through `.schema` directly, matching how the SDK itself handles this object in `framework.py:1515` and `parameters.py:357`. The other dev_mode == "list" paths (`parameters_to_json_schema` for inline `EvalParameters`, empty when no parameters are declared) are preserved. Pylon: #15895 Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/eval-runner.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/eval-runner.py b/scripts/eval-runner.py index 6688f912..d4b243db 100755 --- a/scripts/eval-runner.py +++ b/scripts/eval-runner.py @@ -26,7 +26,7 @@ set_thread_pool_max_workers, ) from braintrust.logger import Dataset, init as init_logger_experiment, parent_context, _internal_get_global_state - from braintrust.parameters import parameters_to_json_schema, validate_parameters + from braintrust.parameters import RemoteEvalParameters, parameters_to_json_schema, validate_parameters from braintrust.util import eprint from braintrust.span_identifier_v4 import parse_parent except Exception as exc: # pragma: no cover - runtime guard @@ -566,8 +566,14 @@ def build_eval_definitions(evaluator_instances: list[EvaluatorInstance]) -> dict for evaluator_instance in evaluator_instances: evaluator = evaluator_instance.evaluator scores = [{"name": getattr(score, "__name__", f"scorer_{i}")} for i, score in enumerate(evaluator.scores)] + if isinstance(evaluator.parameters, RemoteEvalParameters): + parameters_schema: dict[str, Any] = evaluator.parameters.schema or {} + elif evaluator.parameters: + parameters_schema = parameters_to_json_schema(evaluator.parameters) + else: + parameters_schema = {} definitions[evaluator.eval_name] = { - "parameters": parameters_to_json_schema(evaluator.parameters) if evaluator.parameters else {}, + "parameters": parameters_schema, "scores": scores, } return definitions