diff --git a/benchmarks/commit0/eval_infer.py b/benchmarks/commit0/eval_infer.py index 8b45f85ca..8ca7ee54a 100644 --- a/benchmarks/commit0/eval_infer.py +++ b/benchmarks/commit0/eval_infer.py @@ -51,13 +51,16 @@ def process_commit0_results(input_file: str, output_file: str) -> None: "total_instances": 16, "submitted_instances": 16, "completed_instances": 16, + "incomplete_instances": 0, "resolved_instances": 5, "unresolved_instances": 11, "empty_patch_instances": 0, "error_instances": 0, "total_tests": 500, "total_passed_tests": 400, + "submitted_ids": [...], "completed_ids": [...], + "incomplete_ids": [...], "resolved_ids": [...], "unresolved_ids": [...] } @@ -67,6 +70,7 @@ def process_commit0_results(input_file: str, output_file: str) -> None: completed_ids = [] resolved_ids = [] unresolved_ids = [] + incomplete_ids = [] total_tests = 0 total_passed_tests = 0 @@ -118,18 +122,51 @@ def process_commit0_results(input_file: str, output_file: str) -> None: except Exception as e: logger.error(f"Line {line_num}: Unexpected error - {e}") + completed_seen = set(completed_ids) + incomplete_seen = set() + error_path = Path(input_file).with_name(f"{Path(input_file).stem}_errors.jsonl") + if error_path.exists(): + with open(error_path, "r") as error_file: + for line_num, line in enumerate(error_file, 1): + try: + line = line.strip() + if not line: + continue + + data = json.loads(line) + instance_id = data.get("instance_id") + if not instance_id: + logger.warning( + f"Error file line {line_num}: Missing instance_id" + ) + continue + if instance_id in completed_seen or instance_id in incomplete_seen: + continue + + incomplete_ids.append(instance_id) + incomplete_seen.add(instance_id) + except json.JSONDecodeError as e: + logger.error(f"Error file line {line_num}: Invalid JSON - {e}") + except Exception as e: + logger.error(f"Error file line {line_num}: Unexpected error - {e}") + + submitted_ids = completed_ids + incomplete_ids + # Generate report report = { "total_instances": 16, # Fixed as per requirement - "submitted_instances": len(completed_ids), + "submitted_instances": len(submitted_ids), "completed_instances": len(completed_ids), + "incomplete_instances": len(incomplete_ids), "resolved_instances": len(resolved_ids), "unresolved_instances": len(unresolved_ids), "empty_patch_instances": 0, # Always 0 as per requirement - "error_instances": 0, # Always 0 as per requirement + "error_instances": len(incomplete_ids), "total_tests": total_tests, "total_passed_tests": total_passed_tests, + "submitted_ids": submitted_ids, "completed_ids": completed_ids, + "incomplete_ids": incomplete_ids, "resolved_ids": resolved_ids, "unresolved_ids": unresolved_ids, } diff --git a/tests/test_commit0_eval_infer.py b/tests/test_commit0_eval_infer.py index 4f38a1967..7ddc298b0 100644 --- a/tests/test_commit0_eval_infer.py +++ b/tests/test_commit0_eval_infer.py @@ -89,6 +89,40 @@ def test_output_file_naming_with_different_input_name(): ) +def test_error_file_instances_are_included_once(tmp_path): + input_file = tmp_path / "output.jsonl" + output_file = tmp_path / "output.report.json" + completed = { + "instance_id": "completed", + "test_result": { + "eval_result": { + "passed": 1.0, + "num_passed": 10, + "num_tests": 10, + } + }, + } + input_file.write_text(json.dumps(completed) + "\n") + + errors = [ + {"instance_id": "completed", "error": "old timeout"}, + {"instance_id": "failed", "error": "runtime timeout"}, + {"instance_id": "failed", "error": "duplicate retry"}, + ] + error_file = tmp_path / "output_errors.jsonl" + error_file.write_text("".join(json.dumps(row) + "\n" for row in errors)) + + process_commit0_results(str(input_file), str(output_file)) + + report = json.loads(output_file.read_text()) + assert report["submitted_instances"] == 2 + assert report["completed_instances"] == 1 + assert report["incomplete_instances"] == 1 + assert report["error_instances"] == 1 + assert report["submitted_ids"] == ["completed", "failed"] + assert report["incomplete_ids"] == ["failed"] + + def test_output_file_path_derivation(): """Test that Path.with_suffix correctly derives output file name.""" # Test the path derivation logic used in main()