BIRD Mini-Dev run results, hand review and official-script export

evaluation/bird.py

@@ -26,7 +26,10 @@
 (`translate_sqlite`), runs it, and keeps it only if it executes; the rest are excluded and counted,
 never hand-edited. `--sample N` runs a fixed, stratified sample (by database and difficulty).
 
-The questions and results are CC BY-SA 4.0 and are not committed: they live under data/.
+The questions are CC BY-SA 4.0 and are not committed: they live under data/. Run results, which
+quote them, are committed under the same licence (evaluation/results/BIRD_RUNS.md). To check a
+test run with BIRD's own scoring script:
+    python -m evaluation.bird export-official <results.jsonl> <folder>
 """
 
 from __future__ import annotations
@@ -534,6 +537,33 @@ def classify_misses(url: SecretStr, records: dict[str, dict[str, Any]]) -> dict[
     return shapes
 
 
+def export_official(records: dict[str, dict[str, Any]], out: Path, questions: Path = QUESTIONS) -> int:
+    """Write a test run in the input format of BIRD's own evaluation scripts (evaluation_ex.py in
+    bird-bench/mini_dev), so its score can be checked with BIRD's code: predictions as
+    {"0": "SQL\\t----- bird -----\\tdb_id", ...} in question order, the ground truth of `questions`
+    as one "SQL\\tdb_id" line per question, and the questions as JSON lines (the script reads their
+    difficulty). An answer without SQL is written empty, which the script scores as wrong. Line
+    breaks and tabs in SQL become spaces, since both files are line- and tab-separated. Returns the
+    number of answers without SQL."""
+
+    def one_line(text: str) -> str:
+        return text.replace("\r", " ").replace("\n", " ").replace("\t", " ").strip()
+
+    items = json.loads(questions.read_text())
+    predictions, gold, without_sql = {}, [], 0
+    for index, q in enumerate(items):
+        record = records.get(f"B{q['question_id']:04d}", {})
+        answer = one_line(record.get("sql") or "") if record.get("status") == "success" else ""
+        without_sql += not answer
+        predictions[str(index)] = f"{answer}\t----- bird -----\t{q['db_id']}"
+        gold.append(f"{one_line(q['SQL'])}\t{q['db_id']}")
+    out.mkdir(parents=True, exist_ok=True)
+    (out / "predict_mini_dev_PostgreSQL.json").write_text(json.dumps(predictions, indent=1) + "\n")
+    (out / "mini_dev_postgresql_gold.sql").write_text("\n".join(gold) + "\n")
+    (out / "mini_dev_postgresql.jsonl").write_text("".join(json.dumps(q) + "\n" for q in items))
+    return without_sql
+
+
 def _first_line(exc: Exception) -> str:
     return str(exc).strip().splitlines()[0][:300] if str(exc).strip() else exc.__class__.__name__
 
@@ -564,8 +594,20 @@ def main(argv: list[str] | None = None) -> int:
         "misses", help="classify the official-EX misses of a recorded run (re-runs its SQL, no LLM)"
     )
     misses.add_argument("results", type=Path)
+    export = commands.add_parser(
+        "export-official", help="write a test run in the input format of BIRD's evaluation scripts"
+    )
+    export.add_argument("results", type=Path)
+    export.add_argument("out", type=Path, help="folder for the three files")
     args = parser.parse_args(argv)
 
+    if args.command == "export-official":
+        from evaluation.run import load_records  # imported here: evaluation.run imports this module
+
+        without_sql = export_official(load_records(args.results), args.out)
+        print(f"Wrote BIRD's input files to {args.out} ({without_sql} answers without SQL)")
+        return 0
+
     settings = get_settings()
     if settings.database_url is None:
         raise SystemExit("Set DATABASE_URL to the read-only sql_agent account (its server has `bird`).")

evaluation/bird_review.json

@@ -1,16 +1,18 @@
 {
- "about": "Hand review of BIRD Mini-Dev questions whose ground truth looked wrong or ambiguous while checking the misses of run 20261005-103021-bird-database. Judgement, not measurement: the report shows it beside BIRD's official score and never changes that score. BIRD's answers are never edited. One review pass by the project (every 'ground_truth_error' was confirmed by running a corrected query, which returns a different result); not checked independently. Only questions seen so far are listed: an unlisted question is unreviewed, not correct.",
- "reviewed": "2026-10-05",
+ "about": "Hand review of BIRD Mini-Dev questions whose ground truth looked wrong or ambiguous while checking the misses of runs 20261005-103021-bird-database and 20261012-094139-bird-database. Judgement, not measurement: the report shows it beside BIRD's official score and never changes that score. BIRD's answers are never edited. Reviewed by the project in two passes (every 'ground_truth_error' was confirmed by running a corrected query, which returns a different result); not checked independently. Only questions seen so far are listed: an unlisted question is unreviewed, not correct.",
+ "reviewed": "2026-10-13",
  "verdicts": {
   "ground_truth_error": "BIRD's SQL does not answer the question, and a correct answer gives a different result (a bug in the SQL, or rows counted where the question asks for entities).",
   "ground_truth_questionable": "BIRD's SQL contradicts the question or its own hint, but a reader could defend it.",
   "ambiguous": "The question supports two readings that give different results; BIRD picked one."
  },
  "questions": {
+  "B0234": {"verdict": "ground_truth_error", "note": "Missing parentheses around the OR, and atoms _1 / _2 where the hint names atom TR009_12: 1041 instead of 6 connections (3 bonds)."},
   "B0847": {"verdict": "ground_truth_error", "note": "Sorts NULLS FIRST, so it returns one of the six drivers with no Q2 time instead of the fastest."},
   "B0906": {"verdict": "ground_truth_error", "note": "Orders only by year: an arbitrary one of 16 races in the first season, not the first race; points are the standings' running total."},
   "B0944": {"verdict": "ground_truth_error", "note": "The time-string arithmetic yields NULL on PostgreSQL; the expected answer is a single NULL."},
   "B0951": {"verdict": "ground_truth_error", "note": "Asks how many constructors; returns the number of races (2) of the one matching constructor."},
+  "B1107": {"verdict": "ground_truth_error", "note": "The outer ORDER BY date DESC discards the ranking by crossing: it returns the latest record (2016-04-21), not the first date with the highest crossing (2013-02-15)."},
   "B1168": {"verdict": "ground_truth_error", "note": "Returns an extra column (the birthday) that the question does not ask for, so no answer to the question matches exactly."},
   "B1179": {"verdict": "ground_truth_error", "note": "The hint names one antibody column; the SQL returns three."},
   "B1205": {"verdict": "ground_truth_error", "note": "A yes/no question about one patient; returns one row per lab record (67 rows)."},
@@ -26,6 +28,8 @@
   "B1267": {"verdict": "ground_truth_error", "note": "Counts joined rows, not patients: 7 instead of 6."},
   "B1302": {"verdict": "ground_truth_error", "note": "Counts joined rows, not patients: 7 instead of 1."},
   "B1526": {"verdict": "ground_truth_error", "note": "Filters on a different price than the question and hint; no transaction matches either, and the expected answer is a single NULL."},
+  "B0474": {"verdict": "ground_truth_questionable", "note": "The hint says baseSetSize < 10, the question and BIRD's SQL < 100: following the hint gives 0 instead of 30."},
+  "B0486": {"verdict": "ground_truth_questionable", "note": "The hint divides by SUM(convertedManaCost), BIRD's SQL by the number of cards: 2.43% by the hint, 7.74% expected."},
   "B0928": {"verdict": "ground_truth_questionable", "note": "'Ranked first in the Grand Prix' read as fastest-lap rank 1 rather than the winner; also returns names besides the requested reference name."},
   "B0930": {"verdict": "ground_truth_questionable", "note": "The hint says the minimum rank; the SQL uses rank = 1 (the data also has rank 0 as a placeholder)."},
   "B1152": {"verdict": "ground_truth_questionable", "note": "The question asks outpatient to inpatient; the hint and the SQL compute the inverse."},