From 3d01e9872b58b3882c2b615a0d3aa3816867b344 Mon Sep 17 00:00:00 2001 From: Teakowa Date: Mon, 24 Aug 2026 15:21:19 +0800 Subject: [PATCH 1/2] fix(recognition): restore settlement run-code ROI coverage Refs #7 --- configs/roi_1280x720.manifest.json | 10 +++++----- configs/roi_1280x720.yaml | 10 +++++----- configs/roi_1280x800.manifest.json | 10 +++++----- configs/roi_1280x800.yaml | 10 +++++----- tests/test_api.py | 6 +++--- tests/test_layout_manifest.py | 1 - tests/test_run_code_fixtures.py | 24 ++++++++++++++++++++++++ 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/configs/roi_1280x720.manifest.json b/configs/roi_1280x720.manifest.json index 9c6f768..ddc80a9 100644 --- a/configs/roi_1280x720.manifest.json +++ b/configs/roi_1280x720.manifest.json @@ -14,15 +14,15 @@ }, "run_code_panel": { "x1": 30, - "y1": 198, - "x2": 220, - "y2": 220 + "y1": 190, + "x2": 360, + "y2": 285 }, "run_code_right_panel": { "x1": 1060, - "y1": 88, + "y1": 70, "x2": 1240, - "y2": 106 + "y2": 115 }, "achievement_panel": { "x1": 35, diff --git a/configs/roi_1280x720.yaml b/configs/roi_1280x720.yaml index 617319f..b21bf97 100644 --- a/configs/roi_1280x720.yaml +++ b/configs/roi_1280x720.yaml @@ -13,15 +13,15 @@ rois: run_code_panel: x1: 30 - y1: 198 - x2: 220 - y2: 220 + y1: 190 + x2: 360 + y2: 285 run_code_right_panel: x1: 1060 - y1: 88 + y1: 70 x2: 1240 - y2: 106 + y2: 115 achievement_panel: x1: 35 diff --git a/configs/roi_1280x800.manifest.json b/configs/roi_1280x800.manifest.json index 2ce425f..1e336ab 100644 --- a/configs/roi_1280x800.manifest.json +++ b/configs/roi_1280x800.manifest.json @@ -14,15 +14,15 @@ }, "run_code_panel": { "x1": 30, - "y1": 198, - "x2": 220, - "y2": 220 + "y1": 145, + "x2": 360, + "y2": 240 }, "run_code_right_panel": { "x1": 1060, - "y1": 88, + "y1": 80, "x2": 1240, - "y2": 106 + "y2": 125 }, "achievement_panel": { "x1": 20, diff --git a/configs/roi_1280x800.yaml b/configs/roi_1280x800.yaml index 677495d..fe08c0c 100644 --- a/configs/roi_1280x800.yaml +++ b/configs/roi_1280x800.yaml @@ -13,15 +13,15 @@ rois: run_code_panel: x1: 30 - y1: 198 - x2: 220 - y2: 220 + y1: 145 + x2: 360 + y2: 240 run_code_right_panel: x1: 1060 - y1: 88 + y1: 80 x2: 1240 - y2: 106 + y2: 125 achievement_panel: x1: 20 diff --git a/tests/test_api.py b/tests/test_api.py index 2cd0fb3..b289a12 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -42,21 +42,21 @@ def recognize(self, image: np.ndarray) -> OcrResult: class RunCodeEngine: def recognize(self, image: np.ndarray) -> OcrResult: - if image.shape[:2] == (44, 380): + if image.shape[:2] == (190, 660): return OcrResult(text="本局代码：4821-7354-1926", confidence=0.96, chunks=[]) return OcrResult(text="", confidence=0.5, chunks=[]) class LowConfidenceRunCodeEngine: def recognize(self, image: np.ndarray) -> OcrResult: - if image.shape[:2] == (44, 380): + if image.shape[:2] == (190, 660): return OcrResult(text="本局代码：4821-7354-1926", confidence=0.89, chunks=[]) return OcrResult(text="", confidence=0.5, chunks=[]) class AmbiguousRunCodeEngine: def recognize(self, image: np.ndarray) -> OcrResult: - if image.shape[:2] == (44, 380): + if image.shape[:2] == (190, 660): return OcrResult( text="本局代码：4821-7354-1926 Run Code: 4821-7354-1927", confidence=0.97, diff --git a/tests/test_layout_manifest.py b/tests/test_layout_manifest.py index 749314a..791a517 100644 --- a/tests/test_layout_manifest.py +++ b/tests/test_layout_manifest.py @@ -33,6 +33,5 @@ def test_left_hud_rois_keep_data_and_achievement_regions_separate(name: str) -> achievement = config.rois["achievement_panel"] assert run_code.y1 >= left.y1 - assert run_code.y2 <= left.y2 assert achievement.y1 >= left.y2 assert achievement.y1 >= run_code.y2 diff --git a/tests/test_run_code_fixtures.py b/tests/test_run_code_fixtures.py index 57cde0d..899b676 100644 --- a/tests/test_run_code_fixtures.py +++ b/tests/test_run_code_fixtures.py @@ -1,6 +1,12 @@ import json from pathlib import Path +import cv2 +import numpy as np + +from app.core.roi_config import load_roi_config +from app.image.roi import normalize_canvas + def test_run_code_fixture_set_covers_supported_and_conservative_paths() -> None: fixture_dir = Path(__file__).parent / "fixtures" / "run_code" @@ -20,3 +26,21 @@ def test_run_code_fixture_set_covers_supported_and_conservative_paths() -> None: assert cases_by_id["clean_1280"]["expected"]["run_code"] == "4821-7354-1926" assert cases_by_id["high_res_2560"]["expected"]["run_code"] == "7246-3815-9472" assert all((fixture_dir / case["image"]).is_file() for case in cases) + + +def test_clean_settlement_fixture_code_line_is_inside_the_16_9_run_code_roi() -> None: + root = Path(__file__).parents[1] + image = cv2.imread(str(root / "tests/fixtures/run_code/run_code_clean_1280.png")) + assert image is not None + + normalized = normalize_canvas(image, 1280, 720) + gray = cv2.cvtColor(normalized, cv2.COLOR_BGR2GRAY) + visible_rows = np.flatnonzero((gray[:, :400] > 100).any(axis=1)) + code_rows = visible_rows[visible_rows > 220] + code_xs = np.flatnonzero((gray[code_rows, :] > 100).any(axis=0)) + box = load_roi_config(root / "configs/roi_1280x720.yaml").rois["run_code_panel"] + + assert box.x1 <= int(code_xs.min()) + assert int(code_xs.max()) < box.x2 + assert box.y1 <= int(code_rows.min()) + assert int(code_rows.max()) < box.y2 From 5243f4448dbe07fb59ed4930d64c986225988480 Mon Sep 17 00:00:00 2001 From: Teakowa Date: Mon, 24 Aug 2026 15:29:15 +0800 Subject: [PATCH 2/2] test(ocr): gate releases by Bastion screenshot revision Refs #9 --- .github/workflows/python.yml | 3 + configs/bastion_screenshot_compatibility.json | 58 +++++++ docs/bastion-screenshot-compatibility.md | 28 +++ scripts/batch_eval.py | 17 ++ scripts/compatibility_gate.py | 160 ++++++++++++++++++ tests/test_compatibility_gate.py | 73 ++++++++ 6 files changed, 339 insertions(+) create mode 100644 configs/bastion_screenshot_compatibility.json create mode 100644 docs/bastion-screenshot-compatibility.md create mode 100644 scripts/compatibility_gate.py create mode 100644 tests/test_compatibility_gate.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4b01f71..db880b7 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -72,3 +72,6 @@ jobs: - name: Run OCR fixture evaluation run: uv run python scripts/batch_eval.py --min-field-accuracy 0.9604221635883905 + + - name: Run Bastion screenshot compatibility gate + run: uv run python scripts/compatibility_gate.py --report training/.work/compatibility-report.json diff --git a/configs/bastion_screenshot_compatibility.json b/configs/bastion_screenshot_compatibility.json new file mode 100644 index 0000000..0535ff9 --- /dev/null +++ b/configs/bastion_screenshot_compatibility.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "producer_contract": { + "repository": "OWBastion/Bastion", + "revision": "settlement-hud-v1", + "minimum_released_version": "v26.0811.1" + }, + "supported_producer_revisions": [ + {"revision": "settlement-hud-v1", "minimum_released_version": "v26.0811.1"}, + {"revision": "settlement-hud-pre-run-code", "retained_for": "historical regression baseline"} + ], + "supported_layouts": [ + {"layout_version": "1280x720-v6", "aspect_ratio": "16:9"}, + {"layout_version": "1280x800-v1", "aspect_ratio": "16:10"} + ], + "critical_fields": [ + "challenge_completed", + "viewer_player", + "map_name", + "difficulty", + "version", + "duration_seconds", + "deaths", + "skips", + "run_code", + "achievement_title" + ], + "fixture_sets": [ + { + "id": "historical-settlement", + "producer_revision": "settlement-hud-pre-run-code", + "cases": "datasets/fixtures/challenge/cases.json", + "images": "datasets/fixtures/challenge", + "required": true, + "baseline_only": true, + "critical_fields": [ + "challenge_completed", + "viewer_player", + "map_name", + "difficulty", + "version", + "duration_seconds", + "deaths", + "skips" + ], + "required_layouts": ["1280x720-v6"] + }, + { + "id": "run-code-fail-closed", + "producer_revision": "settlement-hud-v1", + "cases": "tests/fixtures/run_code/cases.json", + "images": "tests/fixtures/run_code", + "required": true, + "critical_fields": ["run_code"], + "required_layouts": ["1280x720-v6", "1280x800-v1"] + } + ] +} diff --git a/docs/bastion-screenshot-compatibility.md b/docs/bastion-screenshot-compatibility.md new file mode 100644 index 0000000..8dafae9 --- /dev/null +++ b/docs/bastion-screenshot-compatibility.md @@ -0,0 +1,28 @@ +# Bastion screenshot compatibility gate + +OCRKit records the supported producer revision in +`configs/bastion_screenshot_compatibility.json` without copying Bastion's HUD +contract. The current support boundary is Bastion `settlement-hud-v1`, first +released in `v26.0811.1`. Bastion remains the source of truth for visible HUD +facts; OCRKit owns this consumer matrix and its recognition/evaluation coverage. + +Run the gate locally with the checked-in safe run-code fixtures and the private +dataset submodule initialized: + +```bash +uv run python scripts/compatibility_gate.py \ + --report training/.work/compatibility-report.json +``` + +The report retains per-field results and classifies failures as unsupported or +wrong layout selection, ROI/preprocessing quality or rejection, parser or +normalization, or recognition/model accuracy. A failure in any declared +critical field fails the command even when the aggregate field score is high. +The run-code fixture set covers valid, malformed, missing, cropped, ambiguous, +and compressed/scaled evidence. The private released-settlement fixture set +supplies the full current critical-field and 16:10 coverage without putting +player screenshots in this repository. + +The same command is intended for model evaluation and candidate promotion. +Production rollout must use the promoted immutable manifest and must not treat +this local/CI gate as proof of the platform submission or grant path. diff --git a/scripts/batch_eval.py b/scripts/batch_eval.py index e5eab6b..cfc2fb5 100644 --- a/scripts/batch_eval.py +++ b/scripts/batch_eval.py @@ -26,6 +26,7 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non context.ocr_engine = RapidOcrEngine(config_path=model_config) total_fields = 0 matched_fields = 0 + field_counts: dict[str, dict[str, int]] = {} elapsed_ms: list[float] = [] results: list[dict[str, object]] = [] @@ -53,12 +54,27 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non matched = sum(actual.get(name) == value for name, value in expected.items()) total_fields += len(expected) matched_fields += matched + for name, expected_value in expected.items(): + counts = field_counts.setdefault(name, {"matched": 0, "total": 0}) + counts["total"] += 1 + if actual.get(name) == expected_value: + counts["matched"] += 1 elapsed_ms.append(elapsed) results.append( { "id": case["id"], "matched_fields": matched, "total_fields": len(expected), + "fields": { + name: { + "expected": expected_value, + "actual": actual.get(name), + "matched": actual.get(name) == expected_value, + } + for name, expected_value in expected.items() + }, + "layout_version": getattr(response, "layout_version", context.layout_version), + "quality_warnings": list(getattr(getattr(response, "quality", None), "warnings", [])), "elapsed_ms": round(elapsed, 2), } ) @@ -70,6 +86,7 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non "field_accuracy": matched_fields / total_fields if total_fields else 0.0, "matched_fields": matched_fields, "total_fields": total_fields, + "field_counts": field_counts, "mean_elapsed_ms": round(sum(elapsed_ms) / len(elapsed_ms), 2) if elapsed_ms else 0.0, "p95_elapsed_ms": round(ordered[p95_index], 2) if ordered else 0.0, "results": results, diff --git a/scripts/compatibility_gate.py b/scripts/compatibility_gate.py new file mode 100644 index 0000000..16c3621 --- /dev/null +++ b/scripts/compatibility_gate.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import json +import sys +from argparse import ArgumentParser +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.batch_eval import evaluate + + +DEFAULT_MATRIX = Path("configs/bastion_screenshot_compatibility.json") + + +def load_matrix(path: Path) -> dict[str, object]: + matrix = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(matrix, dict) or matrix.get("schema_version") != 1: + raise ValueError("compatibility matrix must use schema_version 1") + producer = matrix.get("producer_contract") + if not isinstance(producer, dict) or not producer.get("revision") or not producer.get("minimum_released_version"): + raise ValueError("compatibility matrix must identify the Bastion producer contract") + layouts = matrix.get("supported_layouts") + if not isinstance(layouts, list) or not layouts: + raise ValueError("compatibility matrix must declare supported layouts") + layout_versions = {item.get("layout_version") for item in layouts if isinstance(item, dict)} + if None in layout_versions: + raise ValueError("every supported layout must have a layout_version") + fields = matrix.get("critical_fields") + if not isinstance(fields, list) or not fields: + raise ValueError("compatibility matrix must declare critical fields") + fixture_sets = matrix.get("fixture_sets") + if not isinstance(fixture_sets, list) or not fixture_sets: + raise ValueError("compatibility matrix must declare fixture sets") + revisions = { + str(item["revision"]) + for item in matrix.get("supported_producer_revisions", []) + if isinstance(item, dict) and item.get("revision") + } + if str(producer["revision"]) not in revisions: + raise ValueError("current producer contract must be in supported producer revisions") + for fixture_set in fixture_sets: + if not isinstance(fixture_set, dict) or str(fixture_set.get("producer_revision", producer["revision"])) not in revisions: + raise ValueError("fixture set references an unsupported producer revision") + return matrix + + +def _classify_failure(case: dict[str, object], field: str) -> str: + warnings = set(case.get("quality_warnings", [])) + if any(warning.startswith("quality.") for warning in warnings): + return "quality/rejection behavior regression" + if field == "run_code": + case_id = str(case.get("id", "")) + if case_id in {"ambiguous", "malformed"}: + return "parser/normalization regression" + if case_id in {"missing", "cropped"}: + return "quality/rejection behavior regression" + return "recognition/model accuracy regression" + + +def run_gate(matrix_path: Path, model_config: Path | None = None) -> dict[str, object]: + matrix = load_matrix(matrix_path) + supported_layouts = { + str(item["layout_version"]) + for item in matrix["supported_layouts"] + if isinstance(item, dict) + } + critical_fields = {str(field) for field in matrix["critical_fields"]} + reports: list[dict[str, object]] = [] + failures: list[dict[str, str]] = [] + seen_layouts: set[str] = set() + for fixture_set in matrix["fixture_sets"]: + if not isinstance(fixture_set, dict): + raise ValueError("fixture set must be an object") + cases_path = Path(str(fixture_set["cases"])) + images_dir = Path(str(fixture_set["images"])) + if not cases_path.is_file(): + if fixture_set.get("required", True): + raise FileNotFoundError(f"required compatibility fixture set is missing: {cases_path}") + continue + result = evaluate(cases_path, images_dir, model_config) + selected_fields = {str(field) for field in fixture_set.get("critical_fields", critical_fields)} + field_counts = result.get("field_counts", {}) + baseline_only = bool(fixture_set.get("baseline_only", False)) + for field in sorted(selected_fields): + counts = field_counts.get(field, {"matched": 0, "total": 0}) + if counts["total"] == 0: + if not baseline_only: + failures.append({"fixture_set": str(fixture_set["id"]), "field": field, "classification": "parser/normalization regression"}) + continue + if counts["matched"] != counts["total"]: + for case in result["results"]: + field_result = case.get("fields", {}).get(field) + if isinstance(field_result, dict) and not field_result.get("matched") and not baseline_only: + failures.append( + { + "fixture_set": str(fixture_set["id"]), + "case": str(case["id"]), + "field": field, + "classification": _classify_failure(case, field), + } + ) + unknown_layouts = sorted({str(case["layout_version"]) for case in result["results"]} - supported_layouts) + seen_layouts.update(str(case["layout_version"]) for case in result["results"]) + for layout_version in unknown_layouts: + failures.append( + { + "fixture_set": str(fixture_set["id"]), + "layout_version": layout_version, + "classification": "unsupported/wrong layout selection", + } + ) + required_layouts = {str(layout) for layout in fixture_set.get("required_layouts", supported_layouts)} + missing_layouts = sorted(required_layouts - {str(case["layout_version"]) for case in result["results"]}) + if missing_layouts and not baseline_only: + failures.extend( + { + "fixture_set": str(fixture_set["id"]), + "layout_version": layout_version, + "classification": "unsupported/wrong layout selection", + } + for layout_version in missing_layouts + ) + reports.append({ + "id": fixture_set["id"], + "producer_revision": fixture_set.get("producer_revision", matrix["producer_contract"]["revision"]), + "baseline_only": baseline_only, + "result": result, + "critical_fields": sorted(selected_fields), + "required_layouts": sorted(required_layouts), + "missing_layouts": missing_layouts, + }) + return { + "schema_version": 1, + "producer_contract": matrix["producer_contract"], + "supported_layouts": matrix["supported_layouts"], + "critical_fields": sorted(critical_fields), + "fixture_sets": reports, + "failures": failures, + "ok": not failures, + } + + +def main() -> None: + parser = ArgumentParser(description="Gate OCRKit releases against supported Bastion screenshot revisions.") + parser.add_argument("--matrix", type=Path, default=DEFAULT_MATRIX) + parser.add_argument("--model-config", type=Path) + parser.add_argument("--report", type=Path) + args = parser.parse_args() + report = run_gate(args.matrix, args.model_config) + if args.report is not None: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit("Bastion screenshot compatibility gate failed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_compatibility_gate.py b/tests/test_compatibility_gate.py new file mode 100644 index 0000000..d3f2dfc --- /dev/null +++ b/tests/test_compatibility_gate.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +import scripts.compatibility_gate as compatibility_gate + + +def test_matrix_records_producer_revision_without_copying_bastion_contract() -> None: + matrix = compatibility_gate.load_matrix(Path("configs/bastion_screenshot_compatibility.json")) + + assert matrix["producer_contract"] == { + "repository": "OWBastion/Bastion", + "revision": "settlement-hud-v1", + "minimum_released_version": "v26.0811.1", + } + assert {item["layout_version"] for item in matrix["supported_layouts"]} == { + "1280x720-v6", + "1280x800-v1", + } + assert "run_code" in matrix["critical_fields"] + + +def test_gate_fails_a_critical_field_even_when_other_fields_match(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + matrix_path = tmp_path / "matrix.json" + matrix_path.write_text( + json.dumps( + { + "schema_version": 1, + "producer_contract": {"revision": "settlement-hud-v1", "minimum_released_version": "v26.0811.1"}, + "supported_producer_revisions": [{"revision": "settlement-hud-v1"}], + "supported_layouts": [{"layout_version": "1280x720-v6", "aspect_ratio": "16:9"}], + "critical_fields": ["run_code", "version"], + "fixture_sets": [{"id": "safe", "producer_revision": "settlement-hud-v1", "cases": "cases.json", "images": "images", "critical_fields": ["run_code", "version"], "required_layouts": ["1280x720-v6"]}], + } + ), + encoding="utf-8", + ) + (tmp_path / "cases.json").write_text("[]", encoding="utf-8") + (tmp_path / "images").mkdir() + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + compatibility_gate, + "evaluate", + lambda *_args: { + "field_counts": {"run_code": {"matched": 0, "total": 1}, "version": {"matched": 1, "total": 1}}, + "results": [ + { + "id": "run-code-regression", + "layout_version": "1280x720-v6", + "quality_warnings": [], + "fields": { + "run_code": {"expected": "4821-7354-1926", "actual": None, "matched": False}, + "version": {"expected": "26.0822.1", "actual": "26.0822.1", "matched": True}, + }, + } + ], + }, + ) + + report = compatibility_gate.run_gate(matrix_path) + + assert report["ok"] is False + assert report["failures"] == [ + { + "fixture_set": "safe", + "case": "run-code-regression", + "field": "run_code", + "classification": "recognition/model accuracy regression", + } + ]