From ff3331307d7fd4c804698063dfb0e392ea4f1a99 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:16:53 +0000 Subject: [PATCH 01/10] feat(scorers): support multiple python_scorer_* entries in run YAML - Enable key prefix matching for any key under starting with . - Ensure comparator name matches YAML key for clean metric aggregation in , , and . - Add unit tests for multiple python scorers and aggregation in . --- evalbench/scorers/score.py | 18 ++---- evalbench/test/pythonscorer_test.py | 96 +++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 12 deletions(-) diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py index e310b440..82da1d0b 100644 --- a/evalbench/scorers/score.py +++ b/evalbench/scorers/score.py @@ -164,20 +164,14 @@ def compare( ) ) for key, scorer_config in scorers.items(): - if key == "python_scorer": - custom_name = scorer_config.get("scorer_name") - if custom_name and isinstance(custom_name, str): - custom_name = custom_name.strip() - if not custom_name: - script_path = scorer_config.get("script_path") - if script_path and isinstance(script_path, str) and script_path.strip(): - custom_name = os.path.splitext(os.path.basename(script_path))[0].strip() - if not custom_name: - custom_name = key - scorer_config["database_configs"] = experiment_config.get( + if key.startswith("python_scorer"): + if not isinstance(scorer_config, dict): + scorer_config = {} + config_copy = dict(scorer_config) + config_copy["database_configs"] = experiment_config.get( "database_configs", [] ) - comparators.append(pythonscorer.PythonScorer(scorer_config, name=custom_name)) + comparators.append(pythonscorer.PythonScorer(config_copy, name=key)) if "dataform_compile" in scorers: comparators.append( dataformscorer.DataformCompileScorer(scorers["dataform_compile"]) diff --git a/evalbench/test/pythonscorer_test.py b/evalbench/test/pythonscorer_test.py index 6cd7f960..ade7b68e 100644 --- a/evalbench/test/pythonscorer_test.py +++ b/evalbench/test/pythonscorer_test.py @@ -87,6 +87,102 @@ def test_python_scorer_uv_not_found(self, mock_run): self.assertEqual(score, 0.0) self.assertIn("FAIL: 'uv' command not found", reason) + @patch('scorers.pythonscorer.subprocess.run') + def test_multiple_python_scorers_in_compare(self, mock_run): + from scorers import score as score_module + + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = '{"score": 100.0, "reason": "PASS"}' + mock_result.stderr = "" + mock_run.return_value = mock_result + + experiment_config = { + "scorers": { + "python_scorer_accuracy": { + "script_path": "acc_script.py" + }, + "python_scorer_style": { + "script_path": "style_script.py", + "scorer_name": "custom_style_name" + } + } + } + eval_output_item = { + "id": 1, + "nl_prompt": "test", + "golden_sql": "SELECT 1", + "query_type": "SELECT", + "golden_result": "", + "golden_eval_results": "", + "golden_error": "", + "generated_sql": "SELECT 1", + "generated_result": "", + "eval_results": "", + "generated_error": None, + "dialects": "bigquery", + "database": "db", + "job_id": "job123", + } + scoring_results = [] + score_module.compare(eval_output_item, experiment_config, scoring_results, global_models=None) + + comparators = [res["comparator"] for res in scoring_results] + self.assertIn("python_scorer_accuracy", comparators) + self.assertIn("python_scorer_style", comparators) + self.assertEqual(len(scoring_results), 2) + + def test_multiple_python_scorers_aggregation(self): + from reporting.analyzer import analyze_result + + scores = [ + { + "id": "1", + "comparator": "python_scorer_accuracy", + "score": 100, + "generated_sql": "SELECT 1", + "generated_error": None, + }, + { + "id": "2", + "comparator": "python_scorer_accuracy", + "score": 0, + "generated_sql": "SELECT 1", + "generated_error": None, + }, + { + "id": "1", + "comparator": "python_scorer_style", + "score": 100, + "generated_sql": "SELECT 1", + "generated_error": None, + }, + { + "id": "2", + "comparator": "python_scorer_style", + "score": 100, + "generated_sql": "SELECT 1", + "generated_error": None, + }, + ] + experiment_config = { + "scorers": { + "python_scorer_accuracy": {"script_path": "accuracy.py"}, + "python_scorer_style": {"script_path": "style.py", "scorer_name": "custom_style_name"}, + } + } + + _, summary_df = analyze_result(scores, experiment_config) + summary_dict = summary_df.set_index("metric_name").to_dict(orient="index") + + self.assertIn("python_scorer_accuracy", summary_dict) + self.assertEqual(summary_dict["python_scorer_accuracy"]["correct_results_count"], 1) + self.assertEqual(summary_dict["python_scorer_accuracy"]["total_results_count"], 2) + + self.assertIn("python_scorer_style", summary_dict) + self.assertEqual(summary_dict["python_scorer_style"]["correct_results_count"], 2) + self.assertEqual(summary_dict["python_scorer_style"]["total_results_count"], 2) + if __name__ == '__main__': unittest.main() From 722f658a3bcaab92ada3698fb232cf9c64bea1de Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:19:54 +0000 Subject: [PATCH 02/10] docs: add multiple python_scorer examples and parameter documentation --- docs/configs/run-config.md | 2 +- docs/gemini_cli_agent_testing.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/configs/run-config.md b/docs/configs/run-config.md index 6996e32b..818f80ad 100644 --- a/docs/configs/run-config.md +++ b/docs/configs/run-config.md @@ -78,7 +78,7 @@ The `scorers` section defines various scoring strategies to evaluate the quality | `returned_sql` | Optional | Checks that the generated output contains valid SQL code rather than just comments. | | `regexp_matcher` | Optional | Uses regular expressions to determine if the generated query satisfies specific patterns.

**Run Configuration Options:**
- `regexp_string_list` (required): A list of regex patterns to match against the generated query.
- `invert_results` (Optional, default: `False`): When set to true, non-matching queries score 100 and matching queries score 0.
- `match_all_patterns` (Optional, default: `False`): If true, a score of 100 is given only if all regex patterns are matched; otherwise, a match with at least one pattern suffices.
- `match_whole_query` (Optional, default: `False`): When true, forces the pattern to match the entire query rather than a substring. | | `llmrater` | Optional | Compares the execution results of the golden SQL query with those produced by the model. It scores 100 for concrete positive cases, such as mismatches in column names or extra columns in the generated SQL. This scorer requires its own `model_config` for proper operation.

**Run Configuration Options:**
- `hybrid_ground_truth` (Optional, default: `False`): When set to true, if the reference (golden) query execution fails on the target BigQuery engine, it dynamically falls back to resolve the correct reference rows from the local SQLite database file. | -| `python_scorer` | Optional | A generic scorer that executes an external Python script in an isolated sandbox (`uv run --isolated`) to perform custom evaluation logic.

**Run Configuration Options:**
- `script_path` (Required): Path to the Python evaluation script (e.g. `evalbench/scorers/judges/hybrid_xa_judge.py`).
- `scorer_name` (Optional): A custom name for the scorer instance (e.g. `hybrid_cross_db`).

**Included Hybrid Evaluator (`hybrid_xa_judge.py`):**
When set to `evalbench/scorers/judges/hybrid_xa_judge.py`, this operates as a cross-database Execution Accuracy (XA) judge. It compares BigQuery execution results against SQLite references by applying strict cell normalization rules: (1) rounding float values to 4 decimal places, (2) sorting rows lexicographically, (3) stripping trailing `.0` string suffixes, and (4) ignoring column headers. | +| `python_scorer` | Optional | A generic scorer that executes an external Python script in an isolated sandbox (`uv run --isolated`) to perform custom evaluation logic. Multiple Python scorers can be configured under `scorers:` by prefixing keys with `python_scorer` (e.g. `python_scorer_accuracy`, `python_scorer_style`).

**Run Configuration Options:**
- `script_path` (Required): Path to the Python evaluation script (e.g. `evalbench/scorers/judges/hybrid_xa_judge.py`).
- `scorer_name` (Optional): A custom name for the scorer instance.

**Multiple Python Scorers Example:**
```yaml
scorers:
python_scorer_accuracy:
script_path: 'path/to/accuracy_judge.py'
python_scorer_style:
script_path: 'path/to/style_judge.py'
```

**Included Hybrid Evaluator (`hybrid_xa_judge.py`):**
When set to `evalbench/scorers/judges/hybrid_xa_judge.py`, this operates as a cross-database Execution Accuracy (XA) judge. It compares BigQuery execution results against SQLite references by applying strict cell normalization rules: (1) rounding float values to 4 decimal places, (2) sorting rows lexicographically, (3) stripping trailing `.0` string suffixes, and (4) ignoring column headers. | | `recall_match` | Optional | Computes the precision and recall by comparing the generated and expected results, ignoring `None` and duplicate values. The default scoring mode is based on recall, where matching results are compared against the expected outputs regardless of their order. | | `set_match` | Optional | Measures the execution accuracy by comparing the results of the golden query execution with those of the generated query, as defined by the BIRD methodology. | | `exact_match_consistency` | Optional | Evaluates consistency across multiple trials using exact match on execution results. Multiple trials are aggregated at the prompt level using a strict "All-or-Nothing" rule—the prompt is deemed consistent only if ALL trial pairs are consistent. | diff --git a/docs/gemini_cli_agent_testing.md b/docs/gemini_cli_agent_testing.md index 79ee50b3..6c770c9e 100644 --- a/docs/gemini_cli_agent_testing.md +++ b/docs/gemini_cli_agent_testing.md @@ -662,8 +662,15 @@ The `python_scorer` allows you to run arbitrary Python scripts to evaluate agent ```yaml scorers: + # Single Python Scorer python_scorer: script_path: "path/to/your_script.py" + + # Multiple Python Scorers (use python_scorer_* prefix) + python_scorer_accuracy: + script_path: "path/to/accuracy_script.py" + python_scorer_style: + script_path: "path/to/style_script.py" ``` **How it works:** From e9cbaaad4b7a6124236740ed53aa14f6624e214f Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:25:48 +0000 Subject: [PATCH 03/10] feat(scorers): retain custom_name fallbacks and resolve comparator_name in analyzer --- evalbench/reporting/analyzer.py | 20 +++++++++++++++++++- evalbench/scorers/score.py | 11 ++++++++++- evalbench/test/pythonscorer_test.py | 20 ++++++++++---------- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index 4797c666..e4a54005 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -12,6 +12,7 @@ def analyze_one_metric( num_scorers: int = 1, num_prompts: int = None, num_trials: int = None, + experiment_config: dict = None, ) -> dict: """Analyze one metric from dataframe with flexibility.""" num_scorers = max(1, num_scorers) @@ -53,7 +54,23 @@ def analyze_one_metric( .drop_duplicates() ) else: - df_metric = df[df["comparator"] == metric_name] + comparator_name = metric_name + if metric_name.startswith("python_scorer") and experiment_config: + import os + scorers_config = experiment_config.get("scorers", {}) + scorer_config = scorers_config.get(metric_name) + if isinstance(scorer_config, dict): + custom_name = scorer_config.get("scorer_name") + if custom_name and isinstance(custom_name, str): + custom_name = custom_name.strip() + if not custom_name: + script_path = scorer_config.get("script_path") + if script_path and isinstance(script_path, str) and script_path.strip(): + custom_name = os.path.splitext(os.path.basename(script_path))[0].strip() + if custom_name: + comparator_name = custom_name + + df_metric = df[df["comparator"] == comparator_name] if ( "prompt_id" in df_metric.columns @@ -188,6 +205,7 @@ def analyze_result( num_scorers=num_scorers, num_prompts=num_prompts, num_trials=num_trials, + experiment_config=experiment_config, ) summary_scores.append(summary) diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py index 82da1d0b..39d4d91f 100644 --- a/evalbench/scorers/score.py +++ b/evalbench/scorers/score.py @@ -167,11 +167,20 @@ def compare( if key.startswith("python_scorer"): if not isinstance(scorer_config, dict): scorer_config = {} + custom_name = scorer_config.get("scorer_name") + if custom_name and isinstance(custom_name, str): + custom_name = custom_name.strip() + if not custom_name: + script_path = scorer_config.get("script_path") + if script_path and isinstance(script_path, str) and script_path.strip(): + custom_name = os.path.splitext(os.path.basename(script_path))[0].strip() + if not custom_name: + custom_name = key config_copy = dict(scorer_config) config_copy["database_configs"] = experiment_config.get( "database_configs", [] ) - comparators.append(pythonscorer.PythonScorer(config_copy, name=key)) + comparators.append(pythonscorer.PythonScorer(config_copy, name=custom_name)) if "dataform_compile" in scorers: comparators.append( dataformscorer.DataformCompileScorer(scorers["dataform_compile"]) diff --git a/evalbench/test/pythonscorer_test.py b/evalbench/test/pythonscorer_test.py index ade7b68e..fc23a4af 100644 --- a/evalbench/test/pythonscorer_test.py +++ b/evalbench/test/pythonscorer_test.py @@ -100,10 +100,10 @@ def test_multiple_python_scorers_in_compare(self, mock_run): experiment_config = { "scorers": { "python_scorer_accuracy": { - "script_path": "acc_script.py" + "script_path": "path/to/acc_script.py" }, "python_scorer_style": { - "script_path": "style_script.py", + "script_path": "path/to/style_script.py", "scorer_name": "custom_style_name" } } @@ -128,8 +128,8 @@ def test_multiple_python_scorers_in_compare(self, mock_run): score_module.compare(eval_output_item, experiment_config, scoring_results, global_models=None) comparators = [res["comparator"] for res in scoring_results] - self.assertIn("python_scorer_accuracy", comparators) - self.assertIn("python_scorer_style", comparators) + self.assertIn("acc_script", comparators) + self.assertIn("custom_style_name", comparators) self.assertEqual(len(scoring_results), 2) def test_multiple_python_scorers_aggregation(self): @@ -138,28 +138,28 @@ def test_multiple_python_scorers_aggregation(self): scores = [ { "id": "1", - "comparator": "python_scorer_accuracy", + "comparator": "acc_script", "score": 100, "generated_sql": "SELECT 1", "generated_error": None, }, { "id": "2", - "comparator": "python_scorer_accuracy", + "comparator": "acc_script", "score": 0, "generated_sql": "SELECT 1", "generated_error": None, }, { "id": "1", - "comparator": "python_scorer_style", + "comparator": "custom_style_name", "score": 100, "generated_sql": "SELECT 1", "generated_error": None, }, { "id": "2", - "comparator": "python_scorer_style", + "comparator": "custom_style_name", "score": 100, "generated_sql": "SELECT 1", "generated_error": None, @@ -167,8 +167,8 @@ def test_multiple_python_scorers_aggregation(self): ] experiment_config = { "scorers": { - "python_scorer_accuracy": {"script_path": "accuracy.py"}, - "python_scorer_style": {"script_path": "style.py", "scorer_name": "custom_style_name"}, + "python_scorer_accuracy": {"script_path": "path/to/acc_script.py"}, + "python_scorer_style": {"script_path": "path/to/style_script.py", "scorer_name": "custom_style_name"}, } } From 534cca452aef81b62593d5da9a32284c02d18d42 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:28:35 +0000 Subject: [PATCH 04/10] style(analyzer): move import os to top-level and support binary_rubric_scorer prefix in analyzer --- evalbench/reporting/analyzer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index e4a54005..c6cf292f 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -1,6 +1,7 @@ """Analyze accuracy result from dataframe.""" import logging +import os import pandas as pd @@ -56,7 +57,6 @@ def analyze_one_metric( else: comparator_name = metric_name if metric_name.startswith("python_scorer") and experiment_config: - import os scorers_config = experiment_config.get("scorers", {}) scorer_config = scorers_config.get(metric_name) if isinstance(scorer_config, dict): @@ -70,7 +70,10 @@ def analyze_one_metric( if custom_name: comparator_name = custom_name - df_metric = df[df["comparator"] == comparator_name] + if metric_name == "binary_rubric_scorer": + df_metric = df[df["comparator"].astype(str).str.startswith("binary_rubric_scorer")] + else: + df_metric = df[df["comparator"] == comparator_name] if ( "prompt_id" in df_metric.columns From 67acec0fb4c92ce161bb402db2fc71777954b85b Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:29:15 +0000 Subject: [PATCH 05/10] fix(scorers): pass prompt_id to score_dict for prompt-level multi-trial aggregation --- evalbench/scorers/score.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py index 39d4d91f..05ad17a0 100644 --- a/evalbench/scorers/score.py +++ b/evalbench/scorers/score.py @@ -248,5 +248,7 @@ def compare( score_dict["dialects"] = eval_output_item["dialects"] score_dict["database"] = eval_output_item["database"] score_dict["job_id"] = eval_output_item["job_id"] + if "prompt_id" in eval_output_item: + score_dict["prompt_id"] = eval_output_item["prompt_id"] logging.debug("scoring: %d %s %d", score_dict["id"], comp.name, score) scoring_results.append(score_dict) From 314dabf65bcb92199e6d6318e00bad702feaad11 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:30:18 +0000 Subject: [PATCH 06/10] revert: remove binary_rubric_scorer change to isolate in separate PR --- evalbench/reporting/analyzer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index c6cf292f..ec0d447a 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -70,10 +70,7 @@ def analyze_one_metric( if custom_name: comparator_name = custom_name - if metric_name == "binary_rubric_scorer": - df_metric = df[df["comparator"].astype(str).str.startswith("binary_rubric_scorer")] - else: - df_metric = df[df["comparator"] == comparator_name] + df_metric = df[df["comparator"] == comparator_name] if ( "prompt_id" in df_metric.columns From b406941e374b5d0ad3acf2a23a16af635fa49284 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:35:05 +0000 Subject: [PATCH 07/10] refactor(scorers): encapsulate custom_name resolution into get_python_scorer_name helper --- evalbench/reporting/analyzer.py | 12 ++---------- evalbench/scorers/score.py | 11 ++--------- evalbench/scorers/util.py | 23 +++++++++++++++++++++++ evalbench/test/scorer_util_test.py | 20 ++++++++++++++++++++ 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index ec0d447a..42439455 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -3,6 +3,7 @@ import logging import os import pandas as pd +from scorers.util import get_python_scorer_name def analyze_one_metric( @@ -59,16 +60,7 @@ def analyze_one_metric( if metric_name.startswith("python_scorer") and experiment_config: scorers_config = experiment_config.get("scorers", {}) scorer_config = scorers_config.get(metric_name) - if isinstance(scorer_config, dict): - custom_name = scorer_config.get("scorer_name") - if custom_name and isinstance(custom_name, str): - custom_name = custom_name.strip() - if not custom_name: - script_path = scorer_config.get("script_path") - if script_path and isinstance(script_path, str) and script_path.strip(): - custom_name = os.path.splitext(os.path.basename(script_path))[0].strip() - if custom_name: - comparator_name = custom_name + comparator_name = get_python_scorer_name(scorer_config, default_key=metric_name) df_metric = df[df["comparator"] == comparator_name] diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py index 05ad17a0..bfadc1c0 100644 --- a/evalbench/scorers/score.py +++ b/evalbench/scorers/score.py @@ -23,6 +23,7 @@ from scorers import effectivebilledtokens from scorers import binaryrubricscorer from scorers import pythonscorer +from scorers import util from scorers import dataformscorer from scorers import dataformcloudscorer from scorers import dbtscorer @@ -167,15 +168,7 @@ def compare( if key.startswith("python_scorer"): if not isinstance(scorer_config, dict): scorer_config = {} - custom_name = scorer_config.get("scorer_name") - if custom_name and isinstance(custom_name, str): - custom_name = custom_name.strip() - if not custom_name: - script_path = scorer_config.get("script_path") - if script_path and isinstance(script_path, str) and script_path.strip(): - custom_name = os.path.splitext(os.path.basename(script_path))[0].strip() - if not custom_name: - custom_name = key + custom_name = util.get_python_scorer_name(scorer_config, default_key=key) config_copy = dict(scorer_config) config_copy["database_configs"] = experiment_config.get( "database_configs", [] diff --git a/evalbench/scorers/util.py b/evalbench/scorers/util.py index 7b719910..0c8bfc3f 100644 --- a/evalbench/scorers/util.py +++ b/evalbench/scorers/util.py @@ -115,3 +115,26 @@ def filter_conversation_history_json( }) return json.dumps(cleaned_history, indent=2) + + +def get_python_scorer_name(scorer_config: dict, default_key: str = "") -> str: + """Resolves the comparator name for a python_scorer entry. + + Resolution order: + 1. Explicit `scorer_name` parameter in scorer_config. + 2. Basename of `script_path` (without extension). + 3. `default_key` (YAML key string). + """ + if not isinstance(scorer_config, dict): + return default_key + custom_name = scorer_config.get("scorer_name") + if custom_name and isinstance(custom_name, str): + return custom_name.strip() + script_path = scorer_config.get("script_path") + if script_path and isinstance(script_path, str) and script_path.strip(): + import os + base_name = os.path.splitext(os.path.basename(script_path))[0].strip() + if base_name: + return base_name + return default_key + diff --git a/evalbench/test/scorer_util_test.py b/evalbench/test/scorer_util_test.py index b30ea0b0..5faed851 100644 --- a/evalbench/test/scorer_util_test.py +++ b/evalbench/test/scorer_util_test.py @@ -67,3 +67,23 @@ def test_filter_conversation_history_json_with_dict_agent(): agent_1 = json.loads(filtered[0]["agent"]) assert "tool_calls" not in agent_1 assert agent_1["response"] == "Sure, looking." + + +def test_get_python_scorer_name(): + from scorers.util import get_python_scorer_name + + # 1. Explicit scorer_name + cfg1 = {"scorer_name": "custom_name", "script_path": "path/to/script.py"} + assert get_python_scorer_name(cfg1, "default_key") == "custom_name" + + # 2. Script path basename fallback + cfg2 = {"script_path": "path/to/accuracy_judge.py"} + assert get_python_scorer_name(cfg2, "default_key") == "accuracy_judge" + + # 3. Default key fallback + cfg3 = {} + assert get_python_scorer_name(cfg3, "default_key") == "default_key" + + # 4. Invalid config fallback + assert get_python_scorer_name(None, "default_key") == "default_key" + From 44e39ea9e0621e070b8e8a2e92d214492b5fffb1 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:37:22 +0000 Subject: [PATCH 08/10] style(scorers): move import os to top-level in scorers/util.py --- evalbench/scorers/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evalbench/scorers/util.py b/evalbench/scorers/util.py index 0c8bfc3f..d52ed30d 100644 --- a/evalbench/scorers/util.py +++ b/evalbench/scorers/util.py @@ -1,8 +1,9 @@ """Utility functions to aid the scorers.""" from typing import Any -import logging import hashlib +import logging +import os import pickle from util.safe_pickle import safe_pickle_loads @@ -132,7 +133,6 @@ def get_python_scorer_name(scorer_config: dict, default_key: str = "") -> str: return custom_name.strip() script_path = scorer_config.get("script_path") if script_path and isinstance(script_path, str) and script_path.strip(): - import os base_name = os.path.splitext(os.path.basename(script_path))[0].strip() if base_name: return base_name From 2f4d3259c86a108e533e9f8efbd2ce5cf7bf6abb Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Mon, 27 Jul 2026 23:38:46 +0000 Subject: [PATCH 09/10] style(scorers): simplify scorer_config dict fallback in score.py --- evalbench/scorers/score.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/evalbench/scorers/score.py b/evalbench/scorers/score.py index bfadc1c0..daa547bf 100644 --- a/evalbench/scorers/score.py +++ b/evalbench/scorers/score.py @@ -166,8 +166,7 @@ def compare( ) for key, scorer_config in scorers.items(): if key.startswith("python_scorer"): - if not isinstance(scorer_config, dict): - scorer_config = {} + scorer_config = scorer_config if isinstance(scorer_config, dict) else {} custom_name = util.get_python_scorer_name(scorer_config, default_key=key) config_copy = dict(scorer_config) config_copy["database_configs"] = experiment_config.get( From 5f49613e888cb62ac3b0b516c9e247be42002920 Mon Sep 17 00:00:00 2001 From: Saurabh Maurya Date: Tue, 28 Jul 2026 00:12:06 +0000 Subject: [PATCH 10/10] style(analyzer): remove unused import os --- evalbench/reporting/analyzer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index 42439455..572a9ffd 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -1,7 +1,6 @@ """Analyze accuracy result from dataframe.""" import logging -import os import pandas as pd from scorers.util import get_python_scorer_name