From 46bb384c0444b8245e9ad532bf150978e97ea818 Mon Sep 17 00:00:00 2001 From: onatozmenn Date: Fri, 14 Aug 2026 14:50:09 +0300 Subject: [PATCH] fix(swebench): build reports after Modal evaluations --- benchmarks/swebench/eval_infer.py | 12 ++-- benchmarks/swebenchmultilingual/eval_infer.py | 12 ++-- benchmarks/utils/swebench_reports.py | 32 +++++++++ tests/test_swebench_eval_infer.py | 69 +++++++++++++++++++ 4 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 benchmarks/utils/swebench_reports.py diff --git a/benchmarks/swebench/eval_infer.py b/benchmarks/swebench/eval_infer.py index d1ad104ee..39f4796f3 100644 --- a/benchmarks/swebench/eval_infer.py +++ b/benchmarks/swebench/eval_infer.py @@ -26,6 +26,7 @@ from benchmarks.utils.laminar import LaminarService from benchmarks.utils.patch_utils import remove_files_from_patch from benchmarks.utils.report_costs import generate_cost_report +from benchmarks.utils.swebench_reports import ensure_swebench_run_report from openhands.sdk import get_logger @@ -349,10 +350,13 @@ def main() -> None: timeout=args.timeout, ) - # Move report file to input file directory with .report.json extension - # SWE-Bench creates: {MODEL_NAME_OR_PATH}.{run_id}.json - report_filename = f"{MODEL_NAME_OR_PATH}.{args.run_id}.json" - report_path = output_file.parent / report_filename + report_path = ensure_swebench_run_report( + output_file, + dataset=args.dataset, + split=args.split, + run_id=args.run_id, + modal=args.modal, + ) shutil.move(str(report_path), str(dest_report_path)) logger.info(f"Moved report file to: {dest_report_path}") diff --git a/benchmarks/swebenchmultilingual/eval_infer.py b/benchmarks/swebenchmultilingual/eval_infer.py index 3e8ad4f35..87f2c382e 100644 --- a/benchmarks/swebenchmultilingual/eval_infer.py +++ b/benchmarks/swebenchmultilingual/eval_infer.py @@ -22,6 +22,7 @@ from benchmarks.utils.laminar import LaminarService from benchmarks.utils.patch_utils import remove_files_from_patch from benchmarks.utils.report_costs import generate_cost_report +from benchmarks.utils.swebench_reports import ensure_swebench_run_report from openhands.sdk import get_logger @@ -302,10 +303,13 @@ def main() -> None: timeout=args.timeout, ) - # Move report file to input file directory with .report.json extension - # SWE-Bench creates: {MODEL_NAME_OR_PATH}.{run_id}.json - report_filename = f"{MODEL_NAME_OR_PATH}.{args.run_id}.json" - report_path = output_file.parent / report_filename + report_path = ensure_swebench_run_report( + output_file, + dataset=args.dataset, + split=args.split, + run_id=args.run_id, + modal=args.modal, + ) dest_report_path = input_file.with_suffix(".report.json") shutil.move(str(report_path), str(dest_report_path)) diff --git a/benchmarks/utils/swebench_reports.py b/benchmarks/utils/swebench_reports.py new file mode 100644 index 000000000..b45f1e259 --- /dev/null +++ b/benchmarks/utils/swebench_reports.py @@ -0,0 +1,32 @@ +from contextlib import chdir +from pathlib import Path + +from swebench.harness.constants import KEY_INSTANCE_ID +from swebench.harness.reporting import make_run_report +from swebench.harness.utils import get_predictions_from_file, load_swebench_dataset + +from benchmarks.utils.constants import MODEL_NAME_OR_PATH + + +def ensure_swebench_run_report( + predictions_file: Path, + dataset: str, + split: str, + run_id: str, + modal: bool, +) -> Path: + """Return the aggregate report, building it after a Modal run if needed.""" + predictions_file = predictions_file.resolve() + report_path = predictions_file.parent / f"{MODEL_NAME_OR_PATH}.{run_id}.json" + if report_path.exists() or not modal: + return report_path + + prediction_rows = get_predictions_from_file(str(predictions_file), dataset, split) + predictions = {row[KEY_INSTANCE_ID]: row for row in prediction_rows} + full_dataset = load_swebench_dataset(dataset, split) + with chdir(predictions_file.parent): + generated_path = make_run_report(predictions, full_dataset, run_id) + + if generated_path.is_absolute(): + return generated_path + return predictions_file.parent / generated_path diff --git a/tests/test_swebench_eval_infer.py b/tests/test_swebench_eval_infer.py index 29a1df3c3..36b0cbd0c 100644 --- a/tests/test_swebench_eval_infer.py +++ b/tests/test_swebench_eval_infer.py @@ -2,6 +2,7 @@ import json import tempfile +from pathlib import Path from types import SimpleNamespace import pytest @@ -14,6 +15,7 @@ from benchmarks.swebench import apptainer_eval from benchmarks.swebench.eval_infer import convert_to_swebench_format +from benchmarks.utils import swebench_reports from benchmarks.utils.constants import MODEL_NAME_OR_PATH @@ -72,6 +74,73 @@ def test_model_name_or_path_uses_constant(self): assert result["model_name_or_path"] == MODEL_NAME_OR_PATH +def test_modal_report_is_built_when_upstream_does_not_write_one(tmp_path, monkeypatch): + predictions_file = tmp_path / "output.swebench.jsonl" + predictions_file.write_text("{}\n") + prediction_rows = [ + { + "instance_id": "django__django-1", + "model_name_or_path": MODEL_NAME_OR_PATH, + } + ] + predictions = {"django__django-1": prediction_rows[0]} + dataset = [{"instance_id": "django__django-1"}] + original_cwd = Path.cwd() + + monkeypatch.setattr( + swebench_reports, + "get_predictions_from_file", + lambda predictions_path, dataset_name, split: prediction_rows, + ) + monkeypatch.setattr( + swebench_reports, + "load_swebench_dataset", + lambda dataset_name, split: dataset, + ) + + def fake_make_run_report(actual_predictions, full_dataset, run_id): + assert actual_predictions == predictions + assert full_dataset == dataset + assert run_id == "modal-run" + assert Path.cwd() == tmp_path + report_path = Path(f"{MODEL_NAME_OR_PATH}.{run_id}.json") + report_path.write_text("{}") + return report_path + + monkeypatch.setattr(swebench_reports, "make_run_report", fake_make_run_report) + + report_path = swebench_reports.ensure_swebench_run_report( + predictions_file, + dataset="princeton-nlp/SWE-bench_Verified", + split="test", + run_id="modal-run", + modal=True, + ) + + assert report_path == tmp_path / "OpenHands.modal-run.json" + assert report_path.read_text() == "{}" + assert Path.cwd() == original_cwd + + +def test_non_modal_report_keeps_the_upstream_output_path(tmp_path, monkeypatch): + predictions_file = tmp_path / "output.swebench.jsonl" + + def fail_if_called(*args, **kwargs): + raise AssertionError("non-Modal runs should use the upstream report") + + monkeypatch.setattr(swebench_reports, "make_run_report", fail_if_called) + + report_path = swebench_reports.ensure_swebench_run_report( + predictions_file, + dataset="princeton-nlp/SWE-bench_Verified", + split="test", + run_id="docker-run", + modal=False, + ) + + assert report_path == tmp_path / "OpenHands.docker-run.json" + + class TestApptainerEvaluation: """Tests for Apptainer SWE-bench evaluation helpers."""