diff --git a/docs/configs/run-config.md b/docs/configs/run-config.md
index 6996e32b..818f80ad 100644
--- a/docs/configs/run-config.md
+++ b/docs/configs/run-config.md
@@ -78,7 +78,7 @@ The `scorers` section defines various scoring strategies to evaluate the quality
| `returned_sql` | Optional | Checks that the generated output contains valid SQL code rather than just comments. |
| `regexp_matcher` | Optional | Uses regular expressions to determine if the generated query satisfies specific patterns.
**Run Configuration Options:**
- `regexp_string_list` (required): A list of regex patterns to match against the generated query.
- `invert_results` (Optional, default: `False`): When set to true, non-matching queries score 100 and matching queries score 0.
- `match_all_patterns` (Optional, default: `False`): If true, a score of 100 is given only if all regex patterns are matched; otherwise, a match with at least one pattern suffices.
- `match_whole_query` (Optional, default: `False`): When true, forces the pattern to match the entire query rather than a substring. |
| `llmrater` | Optional | Compares the execution results of the golden SQL query with those produced by the model. It scores 100 for concrete positive cases, such as mismatches in column names or extra columns in the generated SQL. This scorer requires its own `model_config` for proper operation.
**Run Configuration Options:**
- `hybrid_ground_truth` (Optional, default: `False`): When set to true, if the reference (golden) query execution fails on the target BigQuery engine, it dynamically falls back to resolve the correct reference rows from the local SQLite database file. |
-| `python_scorer` | Optional | A generic scorer that executes an external Python script in an isolated sandbox (`uv run --isolated`) to perform custom evaluation logic.
**Run Configuration Options:**
- `script_path` (Required): Path to the Python evaluation script (e.g. `evalbench/scorers/judges/hybrid_xa_judge.py`).
- `scorer_name` (Optional): A custom name for the scorer instance (e.g. `hybrid_cross_db`).
**Included Hybrid Evaluator (`hybrid_xa_judge.py`):**
When set to `evalbench/scorers/judges/hybrid_xa_judge.py`, this operates as a cross-database Execution Accuracy (XA) judge. It compares BigQuery execution results against SQLite references by applying strict cell normalization rules: (1) rounding float values to 4 decimal places, (2) sorting rows lexicographically, (3) stripping trailing `.0` string suffixes, and (4) ignoring column headers. |
+| `python_scorer` | Optional | A generic scorer that executes an external Python script in an isolated sandbox (`uv run --isolated`) to perform custom evaluation logic. Multiple Python scorers can be configured under `scorers:` by prefixing keys with `python_scorer` (e.g. `python_scorer_accuracy`, `python_scorer_style`).
**Run Configuration Options:**
- `script_path` (Required): Path to the Python evaluation script (e.g. `evalbench/scorers/judges/hybrid_xa_judge.py`).
- `scorer_name` (Optional): A custom name for the scorer instance.
**Multiple Python Scorers Example:**
```yaml
scorers:
python_scorer_accuracy:
script_path: 'path/to/accuracy_judge.py'
python_scorer_style:
script_path: 'path/to/style_judge.py'
```
**Included Hybrid Evaluator (`hybrid_xa_judge.py`):**
When set to `evalbench/scorers/judges/hybrid_xa_judge.py`, this operates as a cross-database Execution Accuracy (XA) judge. It compares BigQuery execution results against SQLite references by applying strict cell normalization rules: (1) rounding float values to 4 decimal places, (2) sorting rows lexicographically, (3) stripping trailing `.0` string suffixes, and (4) ignoring column headers. |
| `recall_match` | Optional | Computes the precision and recall by comparing the generated and expected results, ignoring `None` and duplicate values. The default scoring mode is based on recall, where matching results are compared against the expected outputs regardless of their order. |
| `set_match` | Optional | Measures the execution accuracy by comparing the results of the golden query execution with those of the generated query, as defined by the BIRD methodology. |
| `exact_match_consistency` | Optional | Evaluates consistency across multiple trials using exact match on execution results. Multiple trials are aggregated at the prompt level using a strict "All-or-Nothing" ruleāthe prompt is deemed consistent only if ALL trial pairs are consistent. |
diff --git a/docs/gemini_cli_agent_testing.md b/docs/gemini_cli_agent_testing.md
index 79ee50b3..6c770c9e 100644
--- a/docs/gemini_cli_agent_testing.md
+++ b/docs/gemini_cli_agent_testing.md
@@ -662,8 +662,15 @@ The `python_scorer` allows you to run arbitrary Python scripts to evaluate agent
```yaml
scorers:
+ # Single Python Scorer
python_scorer:
script_path: "path/to/your_script.py"
+
+ # Multiple Python Scorers (use python_scorer_* prefix)
+ python_scorer_accuracy:
+ script_path: "path/to/accuracy_script.py"
+ python_scorer_style:
+ script_path: "path/to/style_script.py"
```
**How it works:**
diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py
index 4797c666..572a9ffd 100644
--- a/evalbench/reporting/analyzer.py
+++ b/evalbench/reporting/analyzer.py
@@ -2,6 +2,7 @@
import logging
import pandas as pd
+from scorers.util import get_python_scorer_name
def analyze_one_metric(
@@ -12,6 +13,7 @@ def analyze_one_metric(
num_scorers: int = 1,
num_prompts: int = None,
num_trials: int = None,
+ experiment_config: dict = None,
) -> dict:
"""Analyze one metric from dataframe with flexibility."""
num_scorers = max(1, num_scorers)
@@ -53,7 +55,13 @@ def analyze_one_metric(
.drop_duplicates()
)
else:
- df_metric = df[df["comparator"] == metric_name]
+ comparator_name = metric_name
+ if metric_name.startswith("python_scorer") and experiment_config:
+ scorers_config = experiment_config.get("scorers", {})
+ scorer_config = scorers_config.get(metric_name)
+ comparator_name = get_python_scorer_name(scorer_config, default_key=metric_name)
+
+ df_metric = df[df["comparator"] == comparator_name]
if (
"prompt_id" in df_metric.columns
@@ -188,6 +196,7 @@ def analyze_result(
num_scorers=num_scorers,
num_prompts=num_prompts,
num_trials=num_trials,
+ experiment_config=experiment_config,
)
summary_scores.append(summary)
diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py
index e310b440..daa547bf 100644
--- a/evalbench/scorers/score.py
+++ b/evalbench/scorers/score.py
@@ -23,6 +23,7 @@
from scorers import effectivebilledtokens
from scorers import binaryrubricscorer
from scorers import pythonscorer
+from scorers import util
from scorers import dataformscorer
from scorers import dataformcloudscorer
from scorers import dbtscorer
@@ -164,20 +165,14 @@ def compare(
)
)
for key, scorer_config in scorers.items():
- if key == "python_scorer":
- custom_name = scorer_config.get("scorer_name")
- if custom_name and isinstance(custom_name, str):
- custom_name = custom_name.strip()
- if not custom_name:
- script_path = scorer_config.get("script_path")
- if script_path and isinstance(script_path, str) and script_path.strip():
- custom_name = os.path.splitext(os.path.basename(script_path))[0].strip()
- if not custom_name:
- custom_name = key
- scorer_config["database_configs"] = experiment_config.get(
+ if key.startswith("python_scorer"):
+ scorer_config = scorer_config if isinstance(scorer_config, dict) else {}
+ custom_name = util.get_python_scorer_name(scorer_config, default_key=key)
+ config_copy = dict(scorer_config)
+ config_copy["database_configs"] = experiment_config.get(
"database_configs", []
)
- comparators.append(pythonscorer.PythonScorer(scorer_config, name=custom_name))
+ comparators.append(pythonscorer.PythonScorer(config_copy, name=custom_name))
if "dataform_compile" in scorers:
comparators.append(
dataformscorer.DataformCompileScorer(scorers["dataform_compile"])
@@ -245,5 +240,7 @@ def compare(
score_dict["dialects"] = eval_output_item["dialects"]
score_dict["database"] = eval_output_item["database"]
score_dict["job_id"] = eval_output_item["job_id"]
+ if "prompt_id" in eval_output_item:
+ score_dict["prompt_id"] = eval_output_item["prompt_id"]
logging.debug("scoring: %d %s %d", score_dict["id"], comp.name, score)
scoring_results.append(score_dict)
diff --git a/evalbench/scorers/util.py b/evalbench/scorers/util.py
index 7b719910..d52ed30d 100644
--- a/evalbench/scorers/util.py
+++ b/evalbench/scorers/util.py
@@ -1,8 +1,9 @@
"""Utility functions to aid the scorers."""
from typing import Any
-import logging
import hashlib
+import logging
+import os
import pickle
from util.safe_pickle import safe_pickle_loads
@@ -115,3 +116,25 @@ def filter_conversation_history_json(
})
return json.dumps(cleaned_history, indent=2)
+
+
+def get_python_scorer_name(scorer_config: dict, default_key: str = "") -> str:
+ """Resolves the comparator name for a python_scorer entry.
+
+ Resolution order:
+ 1. Explicit `scorer_name` parameter in scorer_config.
+ 2. Basename of `script_path` (without extension).
+ 3. `default_key` (YAML key string).
+ """
+ if not isinstance(scorer_config, dict):
+ return default_key
+ custom_name = scorer_config.get("scorer_name")
+ if custom_name and isinstance(custom_name, str):
+ return custom_name.strip()
+ script_path = scorer_config.get("script_path")
+ if script_path and isinstance(script_path, str) and script_path.strip():
+ base_name = os.path.splitext(os.path.basename(script_path))[0].strip()
+ if base_name:
+ return base_name
+ return default_key
+
diff --git a/evalbench/test/pythonscorer_test.py b/evalbench/test/pythonscorer_test.py
index 6cd7f960..fc23a4af 100644
--- a/evalbench/test/pythonscorer_test.py
+++ b/evalbench/test/pythonscorer_test.py
@@ -87,6 +87,102 @@ def test_python_scorer_uv_not_found(self, mock_run):
self.assertEqual(score, 0.0)
self.assertIn("FAIL: 'uv' command not found", reason)
+ @patch('scorers.pythonscorer.subprocess.run')
+ def test_multiple_python_scorers_in_compare(self, mock_run):
+ from scorers import score as score_module
+
+ mock_result = MagicMock()
+ mock_result.returncode = 0
+ mock_result.stdout = '{"score": 100.0, "reason": "PASS"}'
+ mock_result.stderr = ""
+ mock_run.return_value = mock_result
+
+ experiment_config = {
+ "scorers": {
+ "python_scorer_accuracy": {
+ "script_path": "path/to/acc_script.py"
+ },
+ "python_scorer_style": {
+ "script_path": "path/to/style_script.py",
+ "scorer_name": "custom_style_name"
+ }
+ }
+ }
+ eval_output_item = {
+ "id": 1,
+ "nl_prompt": "test",
+ "golden_sql": "SELECT 1",
+ "query_type": "SELECT",
+ "golden_result": "",
+ "golden_eval_results": "",
+ "golden_error": "",
+ "generated_sql": "SELECT 1",
+ "generated_result": "",
+ "eval_results": "",
+ "generated_error": None,
+ "dialects": "bigquery",
+ "database": "db",
+ "job_id": "job123",
+ }
+ scoring_results = []
+ score_module.compare(eval_output_item, experiment_config, scoring_results, global_models=None)
+
+ comparators = [res["comparator"] for res in scoring_results]
+ self.assertIn("acc_script", comparators)
+ self.assertIn("custom_style_name", comparators)
+ self.assertEqual(len(scoring_results), 2)
+
+ def test_multiple_python_scorers_aggregation(self):
+ from reporting.analyzer import analyze_result
+
+ scores = [
+ {
+ "id": "1",
+ "comparator": "acc_script",
+ "score": 100,
+ "generated_sql": "SELECT 1",
+ "generated_error": None,
+ },
+ {
+ "id": "2",
+ "comparator": "acc_script",
+ "score": 0,
+ "generated_sql": "SELECT 1",
+ "generated_error": None,
+ },
+ {
+ "id": "1",
+ "comparator": "custom_style_name",
+ "score": 100,
+ "generated_sql": "SELECT 1",
+ "generated_error": None,
+ },
+ {
+ "id": "2",
+ "comparator": "custom_style_name",
+ "score": 100,
+ "generated_sql": "SELECT 1",
+ "generated_error": None,
+ },
+ ]
+ experiment_config = {
+ "scorers": {
+ "python_scorer_accuracy": {"script_path": "path/to/acc_script.py"},
+ "python_scorer_style": {"script_path": "path/to/style_script.py", "scorer_name": "custom_style_name"},
+ }
+ }
+
+ _, summary_df = analyze_result(scores, experiment_config)
+ summary_dict = summary_df.set_index("metric_name").to_dict(orient="index")
+
+ self.assertIn("python_scorer_accuracy", summary_dict)
+ self.assertEqual(summary_dict["python_scorer_accuracy"]["correct_results_count"], 1)
+ self.assertEqual(summary_dict["python_scorer_accuracy"]["total_results_count"], 2)
+
+ self.assertIn("python_scorer_style", summary_dict)
+ self.assertEqual(summary_dict["python_scorer_style"]["correct_results_count"], 2)
+ self.assertEqual(summary_dict["python_scorer_style"]["total_results_count"], 2)
+
if __name__ == '__main__':
unittest.main()
diff --git a/evalbench/test/scorer_util_test.py b/evalbench/test/scorer_util_test.py
index b30ea0b0..5faed851 100644
--- a/evalbench/test/scorer_util_test.py
+++ b/evalbench/test/scorer_util_test.py
@@ -67,3 +67,23 @@ def test_filter_conversation_history_json_with_dict_agent():
agent_1 = json.loads(filtered[0]["agent"])
assert "tool_calls" not in agent_1
assert agent_1["response"] == "Sure, looking."
+
+
+def test_get_python_scorer_name():
+ from scorers.util import get_python_scorer_name
+
+ # 1. Explicit scorer_name
+ cfg1 = {"scorer_name": "custom_name", "script_path": "path/to/script.py"}
+ assert get_python_scorer_name(cfg1, "default_key") == "custom_name"
+
+ # 2. Script path basename fallback
+ cfg2 = {"script_path": "path/to/accuracy_judge.py"}
+ assert get_python_scorer_name(cfg2, "default_key") == "accuracy_judge"
+
+ # 3. Default key fallback
+ cfg3 = {}
+ assert get_python_scorer_name(cfg3, "default_key") == "default_key"
+
+ # 4. Invalid config fallback
+ assert get_python_scorer_name(None, "default_key") == "default_key"
+