test(recognition): gate releases by Bastion screenshot revision

.github/workflows/python.yml

@@ -72,3 +72,6 @@ jobs:
 
       - name: Run OCR fixture evaluation
         run: uv run python scripts/batch_eval.py --min-field-accuracy 0.9604221635883905
+
+      - name: Run Bastion screenshot compatibility gate
+        run: uv run python scripts/compatibility_gate.py --report training/.work/compatibility-report.json

configs/bastion_screenshot_compatibility.json

@@ -0,0 +1,58 @@
+{
+  "schema_version": 1,
+  "producer_contract": {
+    "repository": "OWBastion/Bastion",
+    "revision": "settlement-hud-v1",
+    "minimum_released_version": "v26.0811.1"
+  },
+  "supported_producer_revisions": [
+    {"revision": "settlement-hud-v1", "minimum_released_version": "v26.0811.1"},
+    {"revision": "settlement-hud-pre-run-code", "retained_for": "historical regression baseline"}
+  ],
+  "supported_layouts": [
+    {"layout_version": "1280x720-v6", "aspect_ratio": "16:9"},
+    {"layout_version": "1280x800-v1", "aspect_ratio": "16:10"}
+  ],
+  "critical_fields": [
+    "challenge_completed",
+    "viewer_player",
+    "map_name",
+    "difficulty",
+    "version",
+    "duration_seconds",
+    "deaths",
+    "skips",
+    "run_code",
+    "achievement_title"
+  ],
+  "fixture_sets": [
+    {
+      "id": "historical-settlement",
+      "producer_revision": "settlement-hud-pre-run-code",
+      "cases": "datasets/fixtures/challenge/cases.json",
+      "images": "datasets/fixtures/challenge",
+      "required": true,
+      "baseline_only": true,
+      "critical_fields": [
+        "challenge_completed",
+        "viewer_player",
+        "map_name",
+        "difficulty",
+        "version",
+        "duration_seconds",
+        "deaths",
+        "skips"
+      ],
+      "required_layouts": ["1280x720-v6"]
+    },
+    {
+      "id": "run-code-fail-closed",
+      "producer_revision": "settlement-hud-v1",
+      "cases": "tests/fixtures/run_code/cases.json",
+      "images": "tests/fixtures/run_code",
+      "required": true,
+      "critical_fields": ["run_code"],
+      "required_layouts": ["1280x720-v6", "1280x800-v1"]
+    }
+  ]
+}

configs/roi_1280x720.manifest.json

@@ -14,15 +14,15 @@
     },
     "run_code_panel": {
       "x1": 30,
-      "y1": 198,
-      "x2": 220,
-      "y2": 220
+      "y1": 190,
+      "x2": 360,
+      "y2": 285
     },
     "run_code_right_panel": {
       "x1": 1060,
-      "y1": 88,
+      "y1": 70,
       "x2": 1240,
-      "y2": 106
+      "y2": 115
     },
     "achievement_panel": {
       "x1": 35,

configs/roi_1280x720.yaml

@@ -13,15 +13,15 @@ rois:
 
   run_code_panel:
     x1: 30
-    y1: 198
-    x2: 220
-    y2: 220
+    y1: 190
+    x2: 360
+    y2: 285
 
   run_code_right_panel:
     x1: 1060
-    y1: 88
+    y1: 70
     x2: 1240
-    y2: 106
+    y2: 115
 
   achievement_panel:
     x1: 35

configs/roi_1280x800.manifest.json

@@ -14,15 +14,15 @@
     },
     "run_code_panel": {
       "x1": 30,
-      "y1": 198,
-      "x2": 220,
-      "y2": 220
+      "y1": 145,
+      "x2": 360,
+      "y2": 240
     },
     "run_code_right_panel": {
       "x1": 1060,
-      "y1": 88,
+      "y1": 80,
       "x2": 1240,
-      "y2": 106
+      "y2": 125
     },
     "achievement_panel": {
       "x1": 20,

configs/roi_1280x800.yaml

@@ -13,15 +13,15 @@ rois:
 
   run_code_panel:
     x1: 30
-    y1: 198
-    x2: 220
-    y2: 220
+    y1: 145
+    x2: 360
+    y2: 240
 
   run_code_right_panel:
     x1: 1060
-    y1: 88
+    y1: 80
     x2: 1240
-    y2: 106
+    y2: 125
 
   achievement_panel:
     x1: 20

docs/bastion-screenshot-compatibility.md

@@ -0,0 +1,28 @@
+# Bastion screenshot compatibility gate
+
+OCRKit records the supported producer revision in
+`configs/bastion_screenshot_compatibility.json` without copying Bastion's HUD
+contract. The current support boundary is Bastion `settlement-hud-v1`, first
+released in `v26.0811.1`. Bastion remains the source of truth for visible HUD
+facts; OCRKit owns this consumer matrix and its recognition/evaluation coverage.
+
+Run the gate locally with the checked-in safe run-code fixtures and the private
+dataset submodule initialized:
+
+```bash
+uv run python scripts/compatibility_gate.py \
+  --report training/.work/compatibility-report.json
+```
+
+The report retains per-field results and classifies failures as unsupported or
+wrong layout selection, ROI/preprocessing quality or rejection, parser or
+normalization, or recognition/model accuracy. A failure in any declared
+critical field fails the command even when the aggregate field score is high.
+The run-code fixture set covers valid, malformed, missing, cropped, ambiguous,
+and compressed/scaled evidence. The private released-settlement fixture set
+supplies the full current critical-field and 16:10 coverage without putting
+player screenshots in this repository.
+
+The same command is intended for model evaluation and candidate promotion.
+Production rollout must use the promoted immutable manifest and must not treat
+this local/CI gate as proof of the platform submission or grant path.

scripts/batch_eval.py

@@ -26,6 +26,7 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non
         context.ocr_engine = RapidOcrEngine(config_path=model_config)
     total_fields = 0
     matched_fields = 0
+    field_counts: dict[str, dict[str, int]] = {}
     elapsed_ms: list[float] = []
     results: list[dict[str, object]] = []
 
@@ -53,12 +54,27 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non
         matched = sum(actual.get(name) == value for name, value in expected.items())
         total_fields += len(expected)
         matched_fields += matched
+        for name, expected_value in expected.items():
+            counts = field_counts.setdefault(name, {"matched": 0, "total": 0})
+            counts["total"] += 1
+            if actual.get(name) == expected_value:
+                counts["matched"] += 1
         elapsed_ms.append(elapsed)
         results.append(
             {
                 "id": case["id"],
                 "matched_fields": matched,
                 "total_fields": len(expected),
+                "fields": {
+                    name: {
+                        "expected": expected_value,
+                        "actual": actual.get(name),
+                        "matched": actual.get(name) == expected_value,
+                    }
+                    for name, expected_value in expected.items()
+                },
+                "layout_version": getattr(response, "layout_version", context.layout_version),
+                "quality_warnings": list(getattr(getattr(response, "quality", None), "warnings", [])),
                 "elapsed_ms": round(elapsed, 2),
             }
         )
@@ -70,6 +86,7 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non
         "field_accuracy": matched_fields / total_fields if total_fields else 0.0,
         "matched_fields": matched_fields,
         "total_fields": total_fields,
+        "field_counts": field_counts,
         "mean_elapsed_ms": round(sum(elapsed_ms) / len(elapsed_ms), 2) if elapsed_ms else 0.0,
         "p95_elapsed_ms": round(ordered[p95_index], 2) if ordered else 0.0,
         "results": results,

scripts/compatibility_gate.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import json
+import sys
+from argparse import ArgumentParser
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from scripts.batch_eval import evaluate
+
+
+DEFAULT_MATRIX = Path("configs/bastion_screenshot_compatibility.json")
+
+
+def load_matrix(path: Path) -> dict[str, object]:
+    matrix = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(matrix, dict) or matrix.get("schema_version") != 1:
+        raise ValueError("compatibility matrix must use schema_version 1")
+    producer = matrix.get("producer_contract")
+    if not isinstance(producer, dict) or not producer.get("revision") or not producer.get("minimum_released_version"):
+        raise ValueError("compatibility matrix must identify the Bastion producer contract")
+    layouts = matrix.get("supported_layouts")
+    if not isinstance(layouts, list) or not layouts:
+        raise ValueError("compatibility matrix must declare supported layouts")
+    layout_versions = {item.get("layout_version") for item in layouts if isinstance(item, dict)}
+    if None in layout_versions:
+        raise ValueError("every supported layout must have a layout_version")
+    fields = matrix.get("critical_fields")
+    if not isinstance(fields, list) or not fields:
+        raise ValueError("compatibility matrix must declare critical fields")
+    fixture_sets = matrix.get("fixture_sets")
+    if not isinstance(fixture_sets, list) or not fixture_sets:
+        raise ValueError("compatibility matrix must declare fixture sets")
+    revisions = {
+        str(item["revision"])
+        for item in matrix.get("supported_producer_revisions", [])
+        if isinstance(item, dict) and item.get("revision")
+    }
+    if str(producer["revision"]) not in revisions:
+        raise ValueError("current producer contract must be in supported producer revisions")
+    for fixture_set in fixture_sets:
+        if not isinstance(fixture_set, dict) or str(fixture_set.get("producer_revision", producer["revision"])) not in revisions:
+            raise ValueError("fixture set references an unsupported producer revision")
+    return matrix
+
+
+def _classify_failure(case: dict[str, object], field: str) -> str:
+    warnings = set(case.get("quality_warnings", []))
+    if any(warning.startswith("quality.") for warning in warnings):
+        return "quality/rejection behavior regression"
+    if field == "run_code":
+        case_id = str(case.get("id", ""))
+        if case_id in {"ambiguous", "malformed"}:
+            return "parser/normalization regression"
+        if case_id in {"missing", "cropped"}:
+            return "quality/rejection behavior regression"
+    return "recognition/model accuracy regression"
+
+
+def run_gate(matrix_path: Path, model_config: Path | None = None) -> dict[str, object]:
+    matrix = load_matrix(matrix_path)
+    supported_layouts = {
+        str(item["layout_version"])
+        for item in matrix["supported_layouts"]
+        if isinstance(item, dict)
+    }
+    critical_fields = {str(field) for field in matrix["critical_fields"]}
+    reports: list[dict[str, object]] = []
+    failures: list[dict[str, str]] = []
+    seen_layouts: set[str] = set()
+    for fixture_set in matrix["fixture_sets"]:
+        if not isinstance(fixture_set, dict):
+            raise ValueError("fixture set must be an object")
+        cases_path = Path(str(fixture_set["cases"]))
+        images_dir = Path(str(fixture_set["images"]))
+        if not cases_path.is_file():
+            if fixture_set.get("required", True):
+                raise FileNotFoundError(f"required compatibility fixture set is missing: {cases_path}")
+            continue
+        result = evaluate(cases_path, images_dir, model_config)
+        selected_fields = {str(field) for field in fixture_set.get("critical_fields", critical_fields)}
+        field_counts = result.get("field_counts", {})
+        baseline_only = bool(fixture_set.get("baseline_only", False))
+        for field in sorted(selected_fields):
+            counts = field_counts.get(field, {"matched": 0, "total": 0})
+            if counts["total"] == 0:
+                if not baseline_only:
+                    failures.append({"fixture_set": str(fixture_set["id"]), "field": field, "classification": "parser/normalization regression"})
+                continue
+            if counts["matched"] != counts["total"]:
+                for case in result["results"]:
+                    field_result = case.get("fields", {}).get(field)
+                    if isinstance(field_result, dict) and not field_result.get("matched") and not baseline_only:
+                        failures.append(
+                            {
+                                "fixture_set": str(fixture_set["id"]),
+                                "case": str(case["id"]),
+                                "field": field,
+                                "classification": _classify_failure(case, field),
+                            }
+                        )
+        unknown_layouts = sorted({str(case["layout_version"]) for case in result["results"]} - supported_layouts)
+        seen_layouts.update(str(case["layout_version"]) for case in result["results"])
+        for layout_version in unknown_layouts:
+            failures.append(
+                {
+                    "fixture_set": str(fixture_set["id"]),
+                    "layout_version": layout_version,
+                    "classification": "unsupported/wrong layout selection",
+                }
+            )
+        required_layouts = {str(layout) for layout in fixture_set.get("required_layouts", supported_layouts)}
+        missing_layouts = sorted(required_layouts - {str(case["layout_version"]) for case in result["results"]})
+        if missing_layouts and not baseline_only:
+            failures.extend(
+                {
+                    "fixture_set": str(fixture_set["id"]),
+                    "layout_version": layout_version,
+                    "classification": "unsupported/wrong layout selection",
+                }
+                for layout_version in missing_layouts
+            )
+        reports.append({
+            "id": fixture_set["id"],
+            "producer_revision": fixture_set.get("producer_revision", matrix["producer_contract"]["revision"]),
+            "baseline_only": baseline_only,
+            "result": result,
+            "critical_fields": sorted(selected_fields),
+            "required_layouts": sorted(required_layouts),
+            "missing_layouts": missing_layouts,
+        })
+    return {
+        "schema_version": 1,
+        "producer_contract": matrix["producer_contract"],
+        "supported_layouts": matrix["supported_layouts"],
+        "critical_fields": sorted(critical_fields),
+        "fixture_sets": reports,
+        "failures": failures,
+        "ok": not failures,
+    }
+
+
+def main() -> None:
+    parser = ArgumentParser(description="Gate OCRKit releases against supported Bastion screenshot revisions.")
+    parser.add_argument("--matrix", type=Path, default=DEFAULT_MATRIX)
+    parser.add_argument("--model-config", type=Path)
+    parser.add_argument("--report", type=Path)
+    args = parser.parse_args()
+    report = run_gate(args.matrix, args.model_config)
+    if args.report is not None:
+        args.report.parent.mkdir(parents=True, exist_ok=True)
+        args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    if not report["ok"]:
+        raise SystemExit("Bastion screenshot compatibility gate failed")
+
+
+if __name__ == "__main__":
+    main()