diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.yml new file mode 100644 index 0000000..36bbd42 --- /dev/null +++ b/.github/workflows/compatibility.yml @@ -0,0 +1,92 @@ +name: Compatibility + +on: + workflow_dispatch: + schedule: + - cron: "17 3 * * 1" + +permissions: + contents: read + +concurrency: + group: compatibility-${{ github.workflow }} + cancel-in-progress: true + +jobs: + gate: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + token: ${{ github.token }} + + - name: Checkout private OCR fixtures + uses: actions/checkout@v7 + with: + repository: OWBastion/ocrkit-datasets + token: ${{ secrets.OCRKIT_DATASETS_TOKEN }} + path: datasets + fetch-depth: 0 + + - name: Pin OCR fixtures revision + run: | + dataset_commit="$(git rev-parse HEAD:datasets)" + git -C datasets checkout --detach "$dataset_commit" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Install test dependencies + run: uv sync --locked --extra dev + + - name: Run full Bastion screenshot compatibility gate + id: compatibility + shell: bash + run: | + set +e + mkdir -p training/.work + uv run python scripts/compatibility_gate.py \ + --report training/.work/compatibility-report.json \ + > training/.work/compatibility.log 2>&1 + status=$? + { + echo "### Bastion screenshot compatibility gate" + if [[ -f training/.work/compatibility-report.json ]]; then + uv run python - <<'PY' + import json + from pathlib import Path + + report = json.loads(Path("training/.work/compatibility-report.json").read_text()) + failures = report.get("failures", []) + print(f"- status: {'PASS' if report.get('ok') else 'FAIL'}") + print(f"- failures: {len(failures)}") + for failure in failures[:20]: + print(f"- {failure}") + PY + else + echo "- report was not produced" + fi + } >> "$GITHUB_STEP_SUMMARY" + tail -n 80 training/.work/compatibility.log + exit "$status" + + - name: Upload compatibility evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: compatibility-evidence + path: | + training/.work/compatibility-report.json + training/.work/compatibility.log + if-no-files-found: warn diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index db880b7..f1bd028 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -36,6 +36,7 @@ concurrency: jobs: test: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout @@ -43,19 +44,6 @@ jobs: with: token: ${{ github.token }} - - name: Checkout private OCR fixtures - uses: actions/checkout@v7 - with: - repository: OWBastion/ocrkit-datasets - token: ${{ secrets.OCRKIT_DATASETS_TOKEN }} - path: datasets - fetch-depth: 0 - - - name: Pin OCR fixtures revision - run: | - dataset_commit="$(git rev-parse HEAD:datasets)" - git -C datasets checkout --detach "$dataset_commit" - - name: Set up Python uses: actions/setup-python@v5 with: @@ -63,6 +51,9 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: uv.lock - name: Install test dependencies run: uv sync --locked --extra dev @@ -70,8 +61,9 @@ jobs: - name: Run pytest run: uv run pytest -q - - name: Run OCR fixture evaluation - run: uv run python scripts/batch_eval.py --min-field-accuracy 0.9604221635883905 - - - name: Run Bastion screenshot compatibility gate - run: uv run python scripts/compatibility_gate.py --report training/.work/compatibility-report.json + - name: Run public run-code smoke evaluation + run: >- + uv run python scripts/batch_eval.py + --only-run-code + --min-run-code-accuracy 1.0 + --report training/.work/run-code-smoke-report.json diff --git a/README.md b/README.md index f6159be..72b0186 100644 --- a/README.md +++ b/README.md @@ -198,9 +198,10 @@ docker compose up --build -d docker compose ps ``` -Studio updates the stable channel after a fully verified publication. Restart or recreate the OCRKit -container to adopt a newly published or rolled-back channel target. The old environment-variable flow -remains available for initial migration: +Studio publishes a candidate channel after a fully verified publication; compare and explicitly promote +that candidate before the stable channel changes. Restart or recreate the OCRKit container to adopt a +newly promoted or rolled-back channel target. The old environment-variable flow remains available for +initial migration: ```bash docker compose up -d --force-recreate diff --git a/app/model_artifacts/__init__.py b/app/model_artifacts/__init__.py index a26e278..95b6fa0 100644 --- a/app/model_artifacts/__init__.py +++ b/app/model_artifacts/__init__.py @@ -1,4 +1,14 @@ from .channel import ModelReleaseChannel, load_release_channel +from .release import CANDIDATE_CHANNEL_KEY, STABLE_CHANNEL_KEY, compare_manifests from .store import ModelArtifactError, ModelArtifacts, ModelArtifactStore -__all__ = ["ModelArtifactError", "ModelArtifacts", "ModelArtifactStore", "ModelReleaseChannel", "load_release_channel"] +__all__ = [ + "CANDIDATE_CHANNEL_KEY", + "STABLE_CHANNEL_KEY", + "ModelArtifactError", + "ModelArtifacts", + "ModelArtifactStore", + "ModelReleaseChannel", + "compare_manifests", + "load_release_channel", +] diff --git a/app/model_artifacts/release.py b/app/model_artifacts/release.py new file mode 100644 index 0000000..8790f6b --- /dev/null +++ b/app/model_artifacts/release.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +from typing import Any + +from .constants import MODEL_OBJECT_PREFIX + +STABLE_CHANNEL_KEY = f"{MODEL_OBJECT_PREFIX}/channels/stable.json" +CANDIDATE_CHANNEL_KEY = f"{MODEL_OBJECT_PREFIX}/channels/candidate.json" +MIN_FIXTURE_FIELD_ACCURACY = 0.9604221635883905 +MIN_RUN_CODE_ACCURACY = 1.0 + + +def validate_channel_key(channel_key: Any, *, allow_stable: bool = False) -> str: + if not isinstance(channel_key, str): + raise ValueError("model release channel key is invalid") + expected_prefix = f"{MODEL_OBJECT_PREFIX}/channels/" + if not channel_key.startswith(expected_prefix) or not channel_key.endswith(".json"): + raise ValueError("model release channel key is invalid") + if not allow_stable and channel_key == STABLE_CHANNEL_KEY: + raise ValueError("stable channel requires explicit promotion") + return channel_key + + +def parse_channel(payload: bytes, channel_key: str) -> dict[str, Any]: + validate_channel_key(channel_key, allow_stable=True) + try: + data = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("model release channel must be valid JSON") from exc + if not isinstance(data, dict) or data.get("schema_version") != 1 or data.get("model") != "pp-ocrv6-small": + raise ValueError("model release channel has an unsupported schema") + manifest_key = data.get("manifest_key") + if not isinstance(manifest_key, str) or not manifest_key.startswith(f"{MODEL_OBJECT_PREFIX}/") or not manifest_key.endswith("/manifest.json"): + raise ValueError("model release channel manifest key is invalid") + return data + + +def parse_manifest(payload: bytes, manifest_key: str) -> dict[str, Any]: + try: + data = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError("model manifest must be valid JSON") from exc + if not isinstance(data, dict) or data.get("schema_version") != 1 or data.get("model") != "pp-ocrv6-small": + raise ValueError("model manifest has an unsupported schema") + version = data.get("version") + if not isinstance(version, str) or not version or "/" in version: + raise ValueError("model manifest version is invalid") + if manifest_key != f"{MODEL_OBJECT_PREFIX}/{version}/manifest.json": + raise ValueError("model manifest key does not match its version") + return data + + +def _status(value: Any) -> str: + return value.get("status", "missing") if isinstance(value, dict) else "missing" + + +def _metric(report: Any, name: str) -> float | None: + if not isinstance(report, dict): + return None + value = report.get(name) + return float(value) if isinstance(value, (int, float)) else None + + +def _field_metrics(report: Any) -> dict[str, float]: + if not isinstance(report, dict) or not isinstance(report.get("field_metrics"), dict): + return {} + result: dict[str, float] = {} + for name, item in report["field_metrics"].items(): + accuracy = item.get("accuracy") if isinstance(item, dict) else None + if isinstance(name, str) and isinstance(accuracy, (int, float)): + result[name] = float(accuracy) + return result + + +def compare_manifests( + candidate_manifest_key: str, + candidate_manifest: dict[str, Any], + stable_manifest_key: str | None, + stable_manifest: dict[str, Any] | None, +) -> dict[str, Any]: + candidate_evidence = candidate_manifest.get("release_evidence") + candidate_evidence = candidate_evidence if isinstance(candidate_evidence, dict) else {} + evaluation = candidate_evidence.get("evaluation") + evaluation = evaluation if isinstance(evaluation, dict) else {} + fixture = evaluation.get("fixture") + holdout = evaluation.get("holdout") + checks = { + "fixture": _status(fixture) == "passed", + "holdout": _status(holdout) == "passed", + "full_test_suite": _status(candidate_evidence.get("full_test_suite")) == "passed", + "provenance": _status(candidate_evidence.get("provenance")) == "recorded", + "compatibility": _status(candidate_evidence.get("compatibility")) == "passed", + } + reasons = [f"missing or failing {name} evidence" for name, passed in checks.items() if not passed] + fixture_accuracy = _metric(fixture, "field_accuracy") + run_code_accuracy = _metric(fixture.get("run_code") if isinstance(fixture, dict) else None, "field_accuracy") + if fixture_accuracy is None or fixture_accuracy < MIN_FIXTURE_FIELD_ACCURACY: + reasons.append("fixture field accuracy is below the release gate") + if run_code_accuracy is None or run_code_accuracy < MIN_RUN_CODE_ACCURACY: + reasons.append("run-code fixture accuracy is below the release gate") + + stable_evidence = stable_manifest.get("release_evidence", {}) if isinstance(stable_manifest, dict) else {} + stable_eval = stable_evidence.get("evaluation", {}) if isinstance(stable_evidence, dict) else {} + stable_fixture = stable_eval.get("fixture") if isinstance(stable_eval, dict) else None + candidate_fields = _field_metrics(fixture) + stable_fields = _field_metrics(stable_fixture) + field_deltas = { + name: { + "candidate": candidate_fields[name], + "stable": stable_fields[name], + "delta": candidate_fields[name] - stable_fields[name], + } + for name in sorted(candidate_fields.keys() & stable_fields.keys()) + } + + candidate_accuracy = _metric(fixture, "field_accuracy") + stable_accuracy = _metric(stable_fixture, "field_accuracy") + candidate_run_code = _metric(fixture.get("run_code") if isinstance(fixture, dict) else None, "field_accuracy") + stable_run_code = _metric(stable_fixture.get("run_code") if isinstance(stable_fixture, dict) else None, "field_accuracy") + candidate_errors = _metric(fixture, "false_confident_errors") + stable_errors = _metric(stable_fixture, "false_confident_errors") + if candidate_errors is not None and stable_errors is not None and candidate_errors > stable_errors: + reasons.append("candidate has more false-confident errors than stable") + for name, delta in field_deltas.items(): + if delta["delta"] < 0: + reasons.append(f"candidate regresses critical field {name}") + if candidate_accuracy is not None and stable_accuracy is not None and candidate_accuracy < stable_accuracy: + reasons.append("candidate fixture accuracy is below stable") + if candidate_run_code is not None and stable_run_code is not None and candidate_run_code < stable_run_code: + reasons.append("candidate run-code accuracy is below stable") + + return { + "schema_version": 1, + "eligible": not reasons, + "reasons": reasons, + "candidate": { + "manifest_key": candidate_manifest_key, + "version": candidate_manifest["version"], + "evidence": candidate_evidence, + }, + "stable": { + "manifest_key": stable_manifest_key, + "version": stable_manifest.get("version") if isinstance(stable_manifest, dict) else None, + "evidence": stable_evidence, + }, + "comparison": { + "field_accuracy": {"candidate": candidate_accuracy, "stable": stable_accuracy, "delta": candidate_accuracy - stable_accuracy if candidate_accuracy is not None and stable_accuracy is not None else None}, + "run_code_accuracy": {"candidate": candidate_run_code, "stable": stable_run_code, "delta": candidate_run_code - stable_run_code if candidate_run_code is not None and stable_run_code is not None else None}, + "critical_field_deltas": field_deltas, + "false_confident_errors": {"candidate": candidate_errors, "stable": stable_errors}, + }, + } diff --git a/docs/bastion-screenshot-compatibility.md b/docs/bastion-screenshot-compatibility.md index 8dafae9..6152fd5 100644 --- a/docs/bastion-screenshot-compatibility.md +++ b/docs/bastion-screenshot-compatibility.md @@ -23,6 +23,10 @@ and compressed/scaled evidence. The private released-settlement fixture set supplies the full current critical-field and 16:10 coverage without putting player screenshots in this repository. -The same command is intended for model evaluation and candidate promotion. +The same command is intended for model evaluation and candidate promotion. The +full private-corpus gate runs through the manual/nightly `Compatibility` GitHub +Actions workflow and in the model release script; ordinary pull requests run +only the public run-code smoke evaluation so that they do not repeat the full +OCR corpus. Production rollout must use the promoted immutable manifest and must not treat this local/CI gate as proof of the platform submission or grant path. diff --git a/docs/production-deployment.md b/docs/production-deployment.md index de4c73d..8b7cea6 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -11,9 +11,9 @@ Its R2 credential is read-only and may access only the OCRKit model bucket plus The production Compose file defaults `OCRKIT_MODEL_RELEASE_CHANNEL_KEY` to `models/pp-ocrv6-small/channels/stable.json`. Keep `OCRKIT_MODEL_MANIFEST_KEY` only as an optional legacy rollback target; the channel takes -precedence when both are set. After a verified Studio publication updates the channel, recreate -the OCRKit container to download and verify the selected model before serving traffic. No -per-release server environment edit is required. +precedence when both are set. After a verified Studio candidate is explicitly promoted (or a rollback +selects a prior verified manifest), recreate the OCRKit container to download and verify the selected +model before serving traffic. No per-release server environment edit is required. After deployment, verify `https://ocr.owbastion.com/health` anonymously. Recognition endpoints require `Authorization: Bearer ` and are called only by the diff --git a/scripts/batch_eval.py b/scripts/batch_eval.py index cfc2fb5..17f467d 100644 --- a/scripts/batch_eval.py +++ b/scripts/batch_eval.py @@ -27,6 +27,7 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non total_fields = 0 matched_fields = 0 field_counts: dict[str, dict[str, int]] = {} + field_metrics: dict[str, dict[str, int]] = {} elapsed_ms: list[float] = [] results: list[dict[str, object]] = [] @@ -52,13 +53,16 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non actual = response.data.model_dump() if response.data else {} expected = case["expected"] matched = sum(actual.get(name) == value for name, value in expected.items()) - total_fields += len(expected) - matched_fields += matched - for name, expected_value in expected.items(): + for name, value in expected.items(): counts = field_counts.setdefault(name, {"matched": 0, "total": 0}) counts["total"] += 1 - if actual.get(name) == expected_value: + if actual.get(name) == value: counts["matched"] += 1 + metric = field_metrics.setdefault(name, {"matched": 0, "total": 0}) + metric["total"] += 1 + metric["matched"] += actual.get(name) == value + total_fields += len(expected) + matched_fields += matched elapsed_ms.append(elapsed) results.append( { @@ -87,6 +91,10 @@ def evaluate(cases_path: Path, images_dir: Path, model_config: Path | None = Non "matched_fields": matched_fields, "total_fields": total_fields, "field_counts": field_counts, + "field_metrics": { + name: {**metric, "accuracy": metric["matched"] / metric["total"] if metric["total"] else 0.0} + for name, metric in sorted(field_metrics.items()) + }, "mean_elapsed_ms": round(sum(elapsed_ms) / len(elapsed_ms), 2) if elapsed_ms else 0.0, "p95_elapsed_ms": round(ordered[p95_index], 2) if ordered else 0.0, "results": results, @@ -103,14 +111,23 @@ def main() -> None: parser.add_argument("--report", type=Path) parser.add_argument("--min-field-accuracy", type=float) parser.add_argument("--min-run-code-accuracy", type=float, default=1.0) + parser.add_argument( + "--only-run-code", + action="store_true", + help="evaluate only the public run-code fixtures once, without the private challenge corpus", + ) args = parser.parse_args() - result = evaluate(args.cases, args.images_dir, args.model_config) - run_code_result = evaluate(args.run_code_cases, args.run_code_images_dir, args.model_config) - result["run_code"] = run_code_result + if args.only_run_code: + result = evaluate(args.run_code_cases, args.run_code_images_dir, args.model_config) + run_code_result = result + else: + result = evaluate(args.cases, args.images_dir, args.model_config) + run_code_result = evaluate(args.run_code_cases, args.run_code_images_dir, args.model_config) + result["run_code"] = run_code_result if args.report is not None: args.report.parent.mkdir(parents=True, exist_ok=True) args.report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - if args.min_field_accuracy is not None and result["field_accuracy"] < args.min_field_accuracy: + if not args.only_run_code and args.min_field_accuracy is not None and result["field_accuracy"] < args.min_field_accuracy: raise SystemExit( f"fixture field accuracy {result['field_accuracy']:.6f} is below {args.min_field_accuracy:.6f}" ) diff --git a/tests/test_batch_eval.py b/tests/test_batch_eval.py index 05bdbcb..5d8ccdc 100644 --- a/tests/test_batch_eval.py +++ b/tests/test_batch_eval.py @@ -100,3 +100,22 @@ def test_main_rejects_run_code_fixture_accuracy_below_gate(monkeypatch: pytest.M with pytest.raises(SystemExit, match="run-code fixture exact-match accuracy"): batch_eval.main() + + +def test_main_only_run_code_avoids_private_corpus_evaluation(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[Path, Path, Path | None]] = [] + + def evaluate_stub(cases: Path, images: Path, model_config: Path | None = None) -> dict[str, object]: + calls.append((cases, images, model_config)) + return {"field_accuracy": 1.0, "matched_fields": 1, "total_fields": 1} + + monkeypatch.setattr(batch_eval, "evaluate", evaluate_stub) + monkeypatch.setattr( + sys, + "argv", + ["batch_eval.py", "--only-run-code", "--min-run-code-accuracy", "1.0"], + ) + + batch_eval.main() + + assert calls == [(batch_eval.DEFAULT_RUN_CODE_CASES, batch_eval.DEFAULT_RUN_CODE_IMAGES_DIR, None)] diff --git a/tests/test_evaluate_rec_holdout.py b/tests/test_evaluate_rec_holdout.py new file mode 100644 index 0000000..69bd00e --- /dev/null +++ b/tests/test_evaluate_rec_holdout.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import cv2 +import numpy as np +import pytest + +from training.scripts import evaluate_rec_holdout + + +def test_holdout_evaluation_writes_a_promotion_gate_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + crop = tmp_path / "images/holdout/a.png" + crop.parent.mkdir(parents=True) + assert cv2.imwrite(str(crop), np.zeros((4, 4, 3), dtype=np.uint8)) + labels = tmp_path / "holdout.txt" + labels.write_text("images/holdout/a.png\t地图\n", encoding="utf-8") + + class FakeEngine: + def __init__(self, _config: Path) -> None: + pass + + def recognize(self, _image: np.ndarray) -> SimpleNamespace: + return SimpleNamespace(text="地图", confidence=0.99) + + monkeypatch.setattr(evaluate_rec_holdout, "RapidOcrEngine", FakeEngine) + report = evaluate_rec_holdout.evaluate(labels, tmp_path, tmp_path / "rapidocr.yaml", 1.0) + + assert report["status"] == "passed" + assert report["matched"] == 1 + assert report["total"] == 1 + + +def test_holdout_evaluation_rejects_crops_outside_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + labels = tmp_path / "holdout.txt" + labels.write_text("../outside.png\t文本\n", encoding="utf-8") + monkeypatch.setattr(evaluate_rec_holdout, "RapidOcrEngine", lambda _config: object()) + + with pytest.raises(RuntimeError, match="outside the images root"): + evaluate_rec_holdout.evaluate(labels, tmp_path, tmp_path / "rapidocr.yaml", 1.0) diff --git a/tests/test_model_promotion.py b/tests/test_model_promotion.py new file mode 100644 index 0000000..5f7043e --- /dev/null +++ b/tests/test_model_promotion.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import hashlib +import json + +import pytest + +from app.model_artifacts.release import compare_manifests +from training.scripts import promote_model_channel, rollback_model_channel + + +class _Body: + def __init__(self, payload: bytes) -> None: + self.payload = payload + + def read(self) -> bytes: + return self.payload + + +class _Client: + def __init__(self, objects: dict[str, bytes]) -> None: + self.objects = objects + self.puts: list[dict[str, object]] = [] + + def get_object(self, *, Bucket: str, Key: str) -> dict[str, _Body]: # noqa: N803 - boto3 shape + return {"Body": _Body(self.objects[Key])} + + def put_object(self, **kwargs: object) -> None: + self.puts.append(kwargs) + self.objects[str(kwargs["Key"])] = bytes(kwargs["Body"]) + + +def _manifest(version: str, evidence: dict[str, object] | None = None) -> bytes: + payload: dict[str, object] = { + "schema_version": 1, + "model": "pp-ocrv6-small", + "version": version, + "files": { + name: { + "object_key": f"models/pp-ocrv6-small/{version}/{name}", + "sha256": hashlib.sha256(b"artifact").hexdigest(), + "size_bytes": 8, + } + for name in ("det.onnx", "rec.onnx", "rec_dict.txt", "rapidocr.yaml") + }, + } + if evidence is not None: + payload["release_evidence"] = evidence + return json.dumps(payload).encode() + + +def _evidence() -> dict[str, object]: + return { + "schema_version": 1, + "evaluation": { + "fixture": { + "status": "passed", + "field_accuracy": 0.98, + "run_code": {"field_accuracy": 1.0}, + "field_metrics": {"map_name": {"accuracy": 1.0}}, + }, + "holdout": {"status": "passed", "accuracy": 0.97}, + }, + "full_test_suite": {"status": "passed"}, + "provenance": {"status": "recorded", "source": {"snapshot": "s1@v1"}}, + "compatibility": {"status": "passed", "report": {"schema_version": 1, "ok": True}}, + } + + +def _objects(stable_history: list[dict[str, object]] | None = None) -> _Client: + stable = { + "schema_version": 1, + "model": "pp-ocrv6-small", + "manifest_key": "models/pp-ocrv6-small/stable-v1/manifest.json", + } + if stable_history is not None: + stable["history"] = stable_history + return _Client({ + "models/pp-ocrv6-small/channels/candidate.json": json.dumps({ + "schema_version": 1, + "model": "pp-ocrv6-small", + "manifest_key": "models/pp-ocrv6-small/candidate-v2/manifest.json", + }).encode(), + "models/pp-ocrv6-small/channels/stable.json": json.dumps(stable).encode(), + "models/pp-ocrv6-small/candidate-v2/manifest.json": _manifest("candidate-v2", _evidence()), + "models/pp-ocrv6-small/stable-v1/manifest.json": _manifest("stable-v1"), + }) + + +def test_compare_fails_closed_when_candidate_evidence_is_incomplete() -> None: + candidate = json.loads(_manifest("candidate-v2")) + stable = json.loads(_manifest("stable-v1")) + + report = compare_manifests("models/pp-ocrv6-small/candidate-v2/manifest.json", candidate, "models/pp-ocrv6-small/stable-v1/manifest.json", stable) + + assert report["eligible"] is False + assert "missing or failing holdout evidence" in report["reasons"] + assert "missing or failing provenance evidence" in report["reasons"] + + +def test_promote_requires_evidence_and_does_not_write_stable_on_failure() -> None: + client = _objects() + incomplete = json.loads(_manifest("candidate-v2")) + client.objects["models/pp-ocrv6-small/candidate-v2/manifest.json"] = json.dumps(incomplete).encode() + + with pytest.raises(ValueError, match="not eligible"): + promote_model_channel.promote(client, "models", "models/pp-ocrv6-small/channels/candidate.json", "models/pp-ocrv6-small/channels/stable.json") + + assert client.puts == [] + + +def test_promote_writes_only_stable_pointer_after_verification(monkeypatch) -> None: + client = _objects() + monkeypatch.setattr(promote_model_channel, "verify_manifest", lambda _bucket, key: key.rsplit("/", 2)[-2]) + + result = promote_model_channel.promote(client, "models", "models/pp-ocrv6-small/channels/candidate.json", "models/pp-ocrv6-small/channels/stable.json") + + assert result["promoted"] is True + assert len(client.puts) == 1 + stable = json.loads(client.puts[0]["Body"].decode()) + assert stable["manifest_key"].endswith("candidate-v2/manifest.json") + assert stable["previous_manifest_key"].endswith("stable-v1/manifest.json") + + +def test_rollback_requires_history_and_verifies_selected_manifest(monkeypatch) -> None: + previous = "models/pp-ocrv6-small/stable-v0/manifest.json" + client = _objects([{"manifest_key": previous, "action": "promote"}]) + client.objects[previous] = _manifest("stable-v0") + monkeypatch.setattr(rollback_model_channel, "verify_manifest", lambda _bucket, key: key.rsplit("/", 2)[-2]) + + result = rollback_model_channel.rollback(client, "models", "models/pp-ocrv6-small/channels/stable.json", previous) + + assert result["rolled_back"] is True + assert json.loads(client.puts[0]["Body"].decode())["manifest_key"] == previous + + with pytest.raises(ValueError, match="previously verified"): + rollback_model_channel.rollback(client, "models", "models/pp-ocrv6-small/channels/stable.json", "models/pp-ocrv6-small/unknown/manifest.json") diff --git a/tests/test_publish_model_channel.py b/tests/test_publish_model_channel.py index 8b83538..fc50d3d 100644 --- a/tests/test_publish_model_channel.py +++ b/tests/test_publish_model_channel.py @@ -5,6 +5,8 @@ import sys from pathlib import Path +import pytest + def _load_module(): path = Path("training/scripts/publish_model_channel.py") @@ -23,7 +25,7 @@ def put_object(self, **kwargs: object) -> None: self.puts.append(kwargs) -def test_publish_model_channel_writes_stable_pointer_after_release(monkeypatch) -> None: +def test_publish_model_channel_writes_candidate_pointer_after_release(monkeypatch) -> None: module = _load_module() client = StubR2Client() monkeypatch.setattr(module.boto3, "client", lambda *_args, **_kwargs: client) @@ -36,12 +38,25 @@ def test_publish_model_channel_writes_stable_pointer_after_release(monkeypatch) [ "publish_model_channel.py", "--bucket", "models", - "--channel-key", "models/pp-ocrv6-small/channels/stable.json", + "--channel-key", "models/pp-ocrv6-small/channels/candidate.json", "--manifest-key", "models/pp-ocrv6-small/v2/manifest.json", ], ) module.main() - assert client.puts[0]["Key"] == "models/pp-ocrv6-small/channels/stable.json" + assert client.puts[0]["Key"] == "models/pp-ocrv6-small/channels/candidate.json" assert json.loads(client.puts[0]["Body"].decode())["manifest_key"] == "models/pp-ocrv6-small/v2/manifest.json" + + +def test_publish_model_channel_rejects_direct_stable_write(monkeypatch) -> None: + module = _load_module() + monkeypatch.setattr(sys, "argv", [ + "publish_model_channel.py", + "--bucket", "models", + "--channel-key", "models/pp-ocrv6-small/channels/stable.json", + "--manifest-key", "models/pp-ocrv6-small/v2/manifest.json", + ]) + + with pytest.raises(SystemExit, match="explicit promotion"): + module.main() diff --git a/tests/test_release_script.py b/tests/test_release_script.py index 2d3c4fa..af5da90 100644 --- a/tests/test_release_script.py +++ b/tests/test_release_script.py @@ -43,3 +43,6 @@ def test_release_accepts_an_explicit_studio_checkpoint() -> None: assert "--checkpoint" in text assert "publish_model_channel.py" in text assert "--release-channel" in text + assert "candidate.json" in text + assert "stable channel requires explicit promotion" in text + assert "compatibility_gate.py" in text diff --git a/tests/test_studio_app.py b/tests/test_studio_app.py index 242f840..1425522 100644 --- a/tests/test_studio_app.py +++ b/tests/test_studio_app.py @@ -49,6 +49,20 @@ def test_studio_api_serves_local_frontend_and_rejects_unknown_batch(tmp_path: Pa assert client.get("/").text == "
Studio
" +def test_studio_model_promotion_requires_explicit_confirmation(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + frontend = tmp_path / "frontend" + frontend.mkdir() + (frontend / "index.html").write_text("
Studio
", encoding="utf-8") + monkeypatch.setenv("OCRKIT_R2_DEFAULT_BUCKET", "models") + monkeypatch.setenv("OCRKIT_R2_ENDPOINT_URL", "https://example.invalid") + client = TestClient(create_app(tmp_path / "work", frontend)) + + response = client.post("/api/model-release/promote", json={"confirmed": False}) + + assert response.status_code == 422 + assert "确认候选证据" in response.json()["detail"] + + def test_studio_api_imports_image_into_private_batch(tmp_path: Path) -> None: frontend = tmp_path / "frontend" frontend.mkdir() diff --git a/training/README.md b/training/README.md index 41dea04..bb6e70c 100644 --- a/training/README.md +++ b/training/README.md @@ -74,8 +74,9 @@ select/import a finalized platform dataset snapshot (#5) → source-level train/holdout split (ocrkit-split-v1, recorded in provenance) → configure/start or continue Smoke training on the snapshot labels → evaluate the candidate checkpoint -→ publish through the existing immutable release gate -→ rollback by selecting an earlier released manifest/channel target +→ publish an immutable candidate through the existing release gate +→ compare candidate evidence with the current stable manifest +→ explicitly promote to stable, or rollback by selecting an earlier verified manifest ``` `GET /api/snapshots` lists materialized imports; `POST /api/snapshots/import` @@ -442,17 +443,23 @@ batch checkpoint explicitly; this is also available from the command line: ```bash ./training/release_rec_model.sh \ --checkpoint training/.work/studio/batches//runs//checkpoints/best_accuracy \ - --release-channel models/pp-ocrv6-small/channels/stable.json + --holdout-labels training/.work/studio/batches//dataset/labels/holdout.txt \ + --holdout-images-root training/.work/studio/batches//dataset \ + --provenance training/.work/studio/batches//batch.json \ + --release-channel models/pp-ocrv6-small/channels/candidate.json ``` The release command generates an unused UTC version, runs the shared fixture -gate and `uv run pytest -q`, builds a content-hashed manifest, refuses existing -objects, uploads an immutable version under +gate and `uv run pytest -q`, records release evidence, builds a content-hashed +manifest, refuses existing objects, uploads an immutable version under `models/pp-ocrv6-small//`, downloads and verifies the publication with -RapidOCR, then updates the requested release channel. A production container -configured with `OCRKIT_MODEL_RELEASE_CHANNEL_KEY` adopts the channel target -after restart or recreation; a release never overwrites an older model -version. +RapidOCR, then updates only +`models/pp-ocrv6-small/channels/candidate.json`. It evaluates the isolated +holdout crops at the same `364/379` gate, records the holdout result and +provenance in the immutable manifest, and refuses a direct stable +channel write. The Studio or the explicit commands below compare the candidate +with stable before promotion; missing or failing evidence keeps promotion +closed. ## Manual artifact operations @@ -469,16 +476,26 @@ uv run python training/scripts/upload_artifacts.py \ uv run python training/scripts/verify_published_artifact.py \ --bucket "$OCRKIT_R2_DEFAULT_BUCKET" \ --manifest-key models/pp-ocrv6-small//manifest.json +uv run python training/scripts/compare_model_channels.py \ + --bucket "$OCRKIT_R2_DEFAULT_BUCKET" \ + --report training/.work/model-comparison.json +uv run python training/scripts/promote_model_channel.py \ + --bucket "$OCRKIT_R2_DEFAULT_BUCKET" +uv run python training/scripts/rollback_model_channel.py \ + --bucket "$OCRKIT_R2_DEFAULT_BUCKET" \ + --manifest-key models/pp-ocrv6-small//manifest.json ``` `build_manifest.py` fixes the model namespace and records SHA-256 and size for -all four files. Uploads are immutable and must use a new version. Publish a -channel only after download and RapidOCR verification: +all four files, plus the release evidence supplied by the candidate workflow. +Uploads are immutable and must use a new version. Candidate publication is +separate from stable selection; promotion records stable-channel history so +rollback can select a previously verified manifest without retraining. ```bash uv run python training/scripts/publish_model_channel.py \ --bucket "$OCRKIT_R2_DEFAULT_BUCKET" \ - --channel-key models/pp-ocrv6-small/channels/stable.json \ + --channel-key models/pp-ocrv6-small/channels/candidate.json \ --manifest-key models/pp-ocrv6-small//manifest.json ``` @@ -492,8 +509,11 @@ uv run python scripts/batch_eval.py --min-field-accuracy 0.9604221635883905 cargo test --manifest-path rust/Cargo.toml --workspace --locked ``` -The Python workflow runs tests and the fixture gate with the private datasets -revision pinned by the repository submodule. The Rust workflow owns the image +The Python pull-request workflow runs tests and the public run-code smoke +fixtures without checking out private datasets. The manual/nightly +`Compatibility` workflow pins the private datasets revision and runs the full +Bastion screenshot gate, retaining its report and log as an artifact. The model +release script runs the full gate again before publishing a candidate. The Rust workflow owns the image CLI tests and lint. The Docker GHCR workflow intentionally ignores `training/**`, `scripts/**`, `tests/**`, `rust/**`, and `datasets/**` changes; training and model publication remain separate from the production image diff --git a/training/release_rec_model.sh b/training/release_rec_model.sh index b173128..ceebba3 100755 --- a/training/release_rec_model.sh +++ b/training/release_rec_model.sh @@ -14,7 +14,11 @@ work_dir="${root_dir}/training/.work" training_python="${work_dir}/venv/bin/python" paddle2onnx_bin="${work_dir}/venv/bin/paddle2onnx" checkpoint="${work_dir}/checkpoints/rec_pp_ocrv6_small/best_accuracy" -release_channel="${OCRKIT_MODEL_RELEASE_CHANNEL_KEY:-models/pp-ocrv6-small/channels/stable.json}" +release_channel="${OCRKIT_MODEL_CANDIDATE_CHANNEL_KEY:-models/pp-ocrv6-small/channels/candidate.json}" +holdout_report="" +holdout_labels="" +holdout_images_root="" +provenance="" while [[ $# -gt 0 ]]; do case "$1" in --checkpoint) @@ -33,13 +37,53 @@ while [[ $# -gt 0 ]]; do release_channel="$2" shift 2 ;; + --holdout-report) + if [[ $# -lt 2 ]]; then + printf 'usage: %s [--checkpoint ] [--holdout-report ] [--provenance ]\n' "$0" >&2 + exit 2 + fi + holdout_report="$2" + shift 2 + ;; + --holdout-labels) + if [[ $# -lt 2 ]]; then + printf 'usage: %s [--checkpoint ] [--holdout-labels ] [--holdout-images-root ] [--provenance ]\n' "$0" >&2 + exit 2 + fi + holdout_labels="$2" + shift 2 + ;; + --holdout-images-root) + if [[ $# -lt 2 ]]; then + printf 'usage: %s [--checkpoint ] [--holdout-labels ] [--holdout-images-root ] [--provenance ]\n' "$0" >&2 + exit 2 + fi + holdout_images_root="$2" + shift 2 + ;; + --provenance) + if [[ $# -lt 2 ]]; then + printf 'usage: %s [--checkpoint ] [--holdout-report ] [--provenance ]\n' "$0" >&2 + exit 2 + fi + provenance="$2" + shift 2 + ;; *) - printf 'usage: %s [--checkpoint ] [--release-channel ]\n' "$0" >&2 + printf 'usage: %s [--checkpoint ] [--holdout-report ] [--provenance ]\n' "$0" >&2 exit 2 ;; esac done bucket="${OCRKIT_R2_DEFAULT_BUCKET:?set OCRKIT_R2_DEFAULT_BUCKET}" +if [[ "${release_channel}" == "models/pp-ocrv6-small/channels/stable.json" ]]; then + printf 'stable channel requires explicit promotion; publish a candidate instead\n' >&2 + exit 2 +fi +if [[ -n "${holdout_labels}" && -z "${holdout_images_root}" ]]; then + printf '--holdout-images-root is required with --holdout-labels\n' >&2 + exit 2 +fi for path in "${training_python}" "${paddle2onnx_bin}" "${checkpoint}.pdparams"; do if [[ ! -e "${path}" ]]; then @@ -53,8 +97,52 @@ version="$(uv run python training/scripts/next_model_version.py --bucket "${buck artifact_dir="${work_dir}/artifacts/${version}" "${root_dir}/training/evaluate_rec_checkpoint.sh" "${checkpoint}" "${artifact_dir}" -OCRKIT_MODEL_MANIFEST_KEY= uv run pytest -q -uv run python training/scripts/build_manifest.py --artifact-dir "${artifact_dir}" --version "${version}" +if [[ -n "${holdout_labels}" ]]; then + uv run python training/scripts/evaluate_rec_holdout.py \ + --labels "${holdout_labels}" \ + --images-root "${holdout_images_root}" \ + --model-config "${artifact_dir}/rapidocr.yaml" \ + --report "${artifact_dir}/holdout_report.json" +elif [[ -n "${holdout_report}" ]]; then + if [[ ! -f "${holdout_report}" ]]; then + printf 'missing holdout report: %s\n' "${holdout_report}" >&2 + exit 1 + fi + cp "${holdout_report}" "${artifact_dir}/holdout_report.json" +fi +if [[ -f "${artifact_dir}/holdout_report.json" ]]; then + holdout_report="${artifact_dir}/holdout_report.json" +fi + +test_report="${artifact_dir}/full_test_report.txt" +set +e +OCRKIT_MODEL_MANIFEST_KEY= uv run pytest -q >"${test_report}" 2>&1 +test_status=$? +set -e +cat "${test_report}" +if [[ "${test_status}" -ne 0 ]]; then + printf 'full test suite failed; see %s\n' "${test_report}" >&2 + exit "${test_status}" +fi +compatibility_report="${artifact_dir}/compatibility_report.json" +uv run python scripts/compatibility_gate.py --report "${compatibility_report}" +evidence_args=( + --fixture-report "${artifact_dir}/fixture_report.json" + --compatibility-report "${compatibility_report}" + --output "${artifact_dir}/release_evidence.json" +) +if [[ -n "${holdout_report}" ]]; then + evidence_args+=(--holdout-report "${holdout_report}") +fi +if [[ -n "${provenance}" ]]; then + evidence_args+=(--provenance "${provenance}") +fi +uv run python training/scripts/build_release_evidence.py \ + "${evidence_args[@]}" +uv run python training/scripts/build_manifest.py \ + --artifact-dir "${artifact_dir}" \ + --version "${version}" \ + --evidence "${artifact_dir}/release_evidence.json" uv run python training/scripts/upload_artifacts.py --artifact-dir "${artifact_dir}" --bucket "${bucket}" uv run python training/scripts/verify_published_artifact.py \ --bucket "${bucket}" \ @@ -67,4 +155,5 @@ uv run python training/scripts/publish_model_channel.py \ printf 'model_version=%s\n' "${version}" printf 'published_manifest_key=models/pp-ocrv6-small/%s/manifest.json\n' "${version}" printf 'release_channel_key=%s\n' "${release_channel}" +printf 'promotion_action=compare the candidate against stable, then run promote_model_channel.py\n' printf 'deployment_action=recreate the OCRKit container; no per-release env update is required\n' diff --git a/training/scripts/build_manifest.py b/training/scripts/build_manifest.py index dd70250..6f9ae33 100644 --- a/training/scripts/build_manifest.py +++ b/training/scripts/build_manifest.py @@ -29,6 +29,7 @@ def main() -> None: parser.add_argument("--artifact-dir", required=True, type=Path) parser.add_argument("--version", required=True) parser.add_argument("--prefix", default=MODEL_OBJECT_PREFIX) + parser.add_argument("--evidence", type=Path) args = parser.parse_args() if not VERSION_RE.fullmatch(args.version): @@ -56,6 +57,13 @@ def main() -> None: "version": args.version, "files": files, } + if args.evidence is not None: + if not args.evidence.is_file(): + raise SystemExit(f"release evidence does not exist: {args.evidence}") + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(evidence, dict) or evidence.get("schema_version") != 1: + raise SystemExit("release evidence has an unsupported schema") + manifest["release_evidence"] = evidence destination = args.artifact_dir / "manifest.json" destination.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(destination) diff --git a/training/scripts/build_release_evidence.py b/training/scripts/build_release_evidence.py new file mode 100644 index 0000000..aed02f0 --- /dev/null +++ b/training/scripts/build_release_evidence.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +def _read_json(path: Path, label: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SystemExit(f"{label} is not valid JSON: {path}") from exc + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build immutable model release evidence before manifest creation.") + parser.add_argument("--fixture-report", required=True, type=Path) + parser.add_argument("--holdout-report", type=Path) + parser.add_argument("--provenance", type=Path) + parser.add_argument("--compatibility-report", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + fixture = _read_json(args.fixture_report, "fixture report") + if not isinstance(fixture, dict): + raise SystemExit("fixture report must be a JSON object") + holdout: dict[str, Any] = {"status": "missing"} + if args.holdout_report is not None: + holdout_payload = _read_json(args.holdout_report, "holdout report") + if not isinstance(holdout_payload, dict): + raise SystemExit("holdout report must be a JSON object") + holdout = holdout_payload + holdout.setdefault("status", "unverified") + provenance: dict[str, Any] = {"status": "missing"} + if args.provenance is not None: + provenance_payload = _read_json(args.provenance, "release provenance") + provenance = {"status": "recorded", "source": provenance_payload} + compatibility = _read_json(args.compatibility_report, "Bastion screenshot compatibility report") + if not isinstance(compatibility, dict) or compatibility.get("schema_version") != 1: + raise SystemExit("Bastion screenshot compatibility report has an unsupported schema") + if compatibility.get("ok") is not True: + raise SystemExit("Bastion screenshot compatibility gate did not pass") + + evidence = { + "schema_version": 1, + "evaluation": { + "fixture": {**fixture, "status": "passed"}, + "holdout": holdout, + }, + "full_test_suite": {"status": "passed"}, + "provenance": provenance, + "compatibility": {"status": "passed", "report": compatibility}, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(args.output) + + +if __name__ == "__main__": + main() diff --git a/training/scripts/compare_model_channels.py b/training/scripts/compare_model_channels.py new file mode 100644 index 0000000..9f34692 --- /dev/null +++ b/training/scripts/compare_model_channels.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +import boto3 + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from app.model_artifacts.release import compare_manifests, parse_channel, parse_manifest, validate_channel_key + + +def _client() -> Any: + return boto3.client( + "s3", + endpoint_url=os.environ["OCRKIT_R2_ENDPOINT_URL"], + aws_access_key_id=os.environ["OCRKIT_R2_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["OCRKIT_R2_SECRET_ACCESS_KEY"], + region_name=os.getenv("OCRKIT_R2_REGION_NAME", "auto"), + ) + + +def _get_json(client: Any, bucket: str, key: str) -> tuple[bytes, dict[str, Any]]: + body = client.get_object(Bucket=bucket, Key=key)["Body"].read() + return body, json.loads(body) + + +def compare_channels(client: Any, bucket: str, candidate_channel_key: str, stable_channel_key: str) -> dict[str, Any]: + validate_channel_key(candidate_channel_key) + validate_channel_key(stable_channel_key, allow_stable=True) + candidate_channel_bytes, _ = _get_json(client, bucket, candidate_channel_key) + stable_channel_bytes, _ = _get_json(client, bucket, stable_channel_key) + candidate_channel = parse_channel(candidate_channel_bytes, candidate_channel_key) + stable_channel = parse_channel(stable_channel_bytes, stable_channel_key) + candidate_key = str(candidate_channel["manifest_key"]) + stable_key = str(stable_channel["manifest_key"]) + _, candidate_manifest = _get_json(client, bucket, candidate_key) + _, stable_manifest = _get_json(client, bucket, stable_key) + candidate_manifest = parse_manifest(json.dumps(candidate_manifest).encode(), candidate_key) + stable_manifest = parse_manifest(json.dumps(stable_manifest).encode(), stable_key) + report = compare_manifests(candidate_key, candidate_manifest, stable_key, stable_manifest) + report["candidate"]["channel_key"] = candidate_channel_key + report["stable"]["channel_key"] = stable_channel_key + report["stable"]["history"] = stable_channel.get("history", []) + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare the verified candidate channel with the current stable manifest.") + parser.add_argument("--bucket", required=True) + parser.add_argument("--candidate-channel", default="models/pp-ocrv6-small/channels/candidate.json") + parser.add_argument("--stable-channel", default="models/pp-ocrv6-small/channels/stable.json") + parser.add_argument("--report", type=Path) + parser.add_argument("--fail-on-ineligible", action="store_true") + args = parser.parse_args() + report = compare_channels(_client(), args.bucket, args.candidate_channel, args.stable_channel) + if args.report is not None: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if args.fail_on_ineligible and not report["eligible"]: + raise SystemExit("candidate is not eligible for promotion: " + "; ".join(report["reasons"])) + + +if __name__ == "__main__": + main() diff --git a/training/scripts/evaluate_rec_holdout.py b/training/scripts/evaluate_rec_holdout.py new file mode 100644 index 0000000..67211a5 --- /dev/null +++ b/training/scripts/evaluate_rec_holdout.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import argparse +import json +import re +import unicodedata +from pathlib import Path + +import cv2 + +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from app.ocr.rapidocr_engine import RapidOcrEngine + +MIN_HOLDOUT_ACCURACY = 0.9604221635883905 + + +def canonicalize(text: str) -> str: + return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", text).strip()) + + +def evaluate(labels_path: Path, images_root: Path, model_config: Path, min_accuracy: float) -> dict[str, object]: + engine = RapidOcrEngine(model_config) + results: list[dict[str, object]] = [] + matched = 0 + total = 0 + for line in labels_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + relative, expected = line.split("\t", 1) + image_path = (images_root / relative).resolve() + if images_root.resolve() not in image_path.parents or not image_path.is_file(): + raise RuntimeError(f"holdout crop is outside the images root or missing: {relative}") + image = cv2.imread(str(image_path)) + if image is None: + raise RuntimeError(f"cannot read holdout crop: {image_path}") + actual = engine.recognize(image) + expected_text = canonicalize(expected) + actual_text = canonicalize(actual.text) + is_match = actual_text == expected_text + matched += int(is_match) + total += 1 + results.append({ + "crop": relative, + "matched": is_match, + "confidence": actual.confidence, + }) + accuracy = matched / total if total else 0.0 + return { + "schema_version": 1, + "status": "passed" if total > 0 and accuracy >= min_accuracy else "failed", + "accuracy": accuracy, + "matched": matched, + "total": total, + "min_accuracy": min_accuracy, + "results": results, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Evaluate a candidate RapidOCR artifact against isolated holdout crops.") + parser.add_argument("--labels", required=True, type=Path) + parser.add_argument("--images-root", required=True, type=Path) + parser.add_argument("--model-config", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--min-accuracy", type=float, default=MIN_HOLDOUT_ACCURACY) + args = parser.parse_args() + report = evaluate(args.labels, args.images_root, args.model_config, args.min_accuracy) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if report["status"] != "passed": + raise SystemExit(f"holdout accuracy {report['accuracy']:.6f} is below {args.min_accuracy:.6f}") + + +if __name__ == "__main__": + main() diff --git a/training/scripts/promote_model_channel.py b/training/scripts/promote_model_channel.py new file mode 100644 index 0000000..05b7c8e --- /dev/null +++ b/training/scripts/promote_model_channel.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import sys +import tempfile +from pathlib import Path +from typing import Any + +import boto3 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from app.model_artifacts.release import STABLE_CHANNEL_KEY, parse_channel, validate_channel_key +from app.model_artifacts.store import ModelArtifactStore +from app.ocr.rapidocr_engine import RapidOcrEngine +from app.storage.r2_client import R2ObjectStore +from training.scripts.compare_model_channels import compare_channels + + +def _client() -> Any: + return boto3.client( + "s3", + endpoint_url=os.environ["OCRKIT_R2_ENDPOINT_URL"], + aws_access_key_id=os.environ["OCRKIT_R2_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["OCRKIT_R2_SECRET_ACCESS_KEY"], + region_name=os.getenv("OCRKIT_R2_REGION_NAME", "auto"), + ) + + +def verify_manifest(bucket: str, manifest_key: str) -> str: + store = R2ObjectStore.from_settings( + endpoint_url=os.environ["OCRKIT_R2_ENDPOINT_URL"], + access_key_id=os.environ["OCRKIT_R2_ACCESS_KEY_ID"], + secret_access_key=os.environ["OCRKIT_R2_SECRET_ACCESS_KEY"], + region_name=os.getenv("OCRKIT_R2_REGION_NAME", "auto"), + default_bucket=bucket, + allowed_buckets_raw=bucket, + read_timeout_seconds=30, + ) + with tempfile.TemporaryDirectory(prefix="ocrkit-model-promote-") as cache_dir: + artifacts = ModelArtifactStore(store, bucket, Path(cache_dir)).prepare(manifest_key) + RapidOcrEngine(artifacts.rapidocr_config_path) + return artifacts.version + + +def _write_stable(client: Any, bucket: str, stable_channel_key: str, manifest_key: str, previous: dict[str, Any]) -> None: + history = previous.get("history") if isinstance(previous.get("history"), list) else [] + previous_key = previous.get("manifest_key") + if isinstance(previous_key, str) and previous_key != manifest_key: + history = [*history, {"manifest_key": previous_key, "verified_at": previous.get("updated_at"), "action": previous.get("action", "release")}] + payload = { + "schema_version": 1, + "model": "pp-ocrv6-small", + "manifest_key": manifest_key, + "previous_manifest_key": previous_key, + "history": history, + "action": "promote", + "updated_at": dt.datetime.now(dt.timezone.utc).isoformat(), + } + client.put_object( + Bucket=bucket, + Key=stable_channel_key, + Body=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode(), + ContentType="application/json", + ) + + +def promote(client: Any, bucket: str, candidate_channel_key: str, stable_channel_key: str) -> dict[str, Any]: + validate_channel_key(candidate_channel_key) + validate_channel_key(stable_channel_key, allow_stable=True) + report = compare_channels(client, bucket, candidate_channel_key, stable_channel_key) + if not report["eligible"]: + raise ValueError("candidate is not eligible for promotion: " + "; ".join(report["reasons"])) + candidate_channel = parse_channel( + client.get_object(Bucket=bucket, Key=candidate_channel_key)["Body"].read(), candidate_channel_key + ) + stable_payload = client.get_object(Bucket=bucket, Key=stable_channel_key)["Body"].read() + stable_channel = parse_channel(stable_payload, stable_channel_key) + candidate_key = str(candidate_channel["manifest_key"]) + stable_key = str(stable_channel["manifest_key"]) + verify_manifest(bucket, candidate_key) + verify_manifest(bucket, stable_key) + _write_stable(client, bucket, stable_channel_key, candidate_key, stable_channel) + return {"promoted": True, "candidate_manifest_key": candidate_key, "previous_manifest_key": stable_key, "comparison": report} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Explicitly promote an eligible verified candidate to stable.") + parser.add_argument("--bucket", required=True) + parser.add_argument("--candidate-channel", default="models/pp-ocrv6-small/channels/candidate.json") + parser.add_argument("--stable-channel", default=STABLE_CHANNEL_KEY) + args = parser.parse_args() + result = promote(_client(), args.bucket, args.candidate_channel, args.stable_channel) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/training/scripts/publish_model_channel.py b/training/scripts/publish_model_channel.py index 05f9537..ea719de 100644 --- a/training/scripts/publish_model_channel.py +++ b/training/scripts/publish_model_channel.py @@ -12,6 +12,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from app.model_artifacts.constants import MODEL_OBJECT_PREFIX +from app.model_artifacts.release import validate_channel_key def main() -> None: @@ -20,9 +21,10 @@ def main() -> None: parser.add_argument("--channel-key", required=True) parser.add_argument("--manifest-key", required=True) args = parser.parse_args() - channel_prefix = f"{MODEL_OBJECT_PREFIX}/channels/" - if not args.channel_key.startswith(channel_prefix) or not args.channel_key.endswith(".json"): - raise SystemExit("release channel key is invalid") + try: + validate_channel_key(args.channel_key) + except ValueError as exc: + raise SystemExit(str(exc)) from exc if not args.manifest_key.startswith(f"{MODEL_OBJECT_PREFIX}/") or not args.manifest_key.endswith("/manifest.json"): raise SystemExit("model manifest key is invalid") client = boto3.client( diff --git a/training/scripts/rollback_model_channel.py b/training/scripts/rollback_model_channel.py new file mode 100644 index 0000000..14c6531 --- /dev/null +++ b/training/scripts/rollback_model_channel.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import sys +from pathlib import Path +from typing import Any + +import boto3 + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from app.model_artifacts.release import STABLE_CHANNEL_KEY, parse_channel, parse_manifest, validate_channel_key +from training.scripts.promote_model_channel import verify_manifest + + +def _client() -> Any: + return boto3.client( + "s3", + endpoint_url=os.environ["OCRKIT_R2_ENDPOINT_URL"], + aws_access_key_id=os.environ["OCRKIT_R2_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["OCRKIT_R2_SECRET_ACCESS_KEY"], + region_name=os.getenv("OCRKIT_R2_REGION_NAME", "auto"), + ) + + +def rollback(client: Any, bucket: str, stable_channel_key: str, manifest_key: str) -> dict[str, Any]: + validate_channel_key(stable_channel_key, allow_stable=True) + stable_payload = client.get_object(Bucket=bucket, Key=stable_channel_key)["Body"].read() + stable = parse_channel(stable_payload, stable_channel_key) + current_key = str(stable["manifest_key"]) + history = stable.get("history") if isinstance(stable.get("history"), list) else [] + allowed = {entry.get("manifest_key") for entry in history if isinstance(entry, dict)} + if manifest_key == current_key or manifest_key not in allowed: + raise ValueError("rollback target is not a previously verified stable manifest") + manifest_payload = client.get_object(Bucket=bucket, Key=manifest_key)["Body"].read() + parse_manifest(manifest_payload, manifest_key) + verify_manifest(bucket, manifest_key) + payload = { + "schema_version": 1, + "model": "pp-ocrv6-small", + "manifest_key": manifest_key, + "previous_manifest_key": current_key, + "history": [*history, {"manifest_key": current_key, "verified_at": stable.get("updated_at"), "action": "rollback-source"}], + "action": "rollback", + "updated_at": dt.datetime.now(dt.timezone.utc).isoformat(), + } + client.put_object( + Bucket=bucket, + Key=stable_channel_key, + Body=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode(), + ContentType="application/json", + ) + return {"rolled_back": True, "manifest_key": manifest_key, "previous_manifest_key": current_key} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Repoint stable to a previously verified manifest.") + parser.add_argument("--bucket", required=True) + parser.add_argument("--manifest-key", required=True) + parser.add_argument("--candidate-channel") + parser.add_argument("--stable-channel", default=STABLE_CHANNEL_KEY) + args = parser.parse_args() + print(json.dumps(rollback(_client(), args.bucket, args.stable_channel, args.manifest_key), ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/training/studio/app.py b/training/studio/app.py index dd5f1cd..286b9fc 100644 --- a/training/studio/app.py +++ b/training/studio/app.py @@ -7,6 +7,7 @@ import re import shutil import subprocess +import sys import tempfile from datetime import UTC, datetime from pathlib import Path @@ -75,6 +76,15 @@ class PublishStart(BaseModel): confirmed: bool = False +class ReleaseAction(BaseModel): + confirmed: bool = False + + +class RollbackAction(BaseModel): + confirmed: bool = False + manifest_key: str = Field(..., min_length=1, max_length=256) + + class RemoteSourceSelection(BaseModel): keys: list[str] = Field(min_length=1, max_length=200) holdout_ratio: float = Field(default=0.2, ge=0, lt=1) @@ -352,6 +362,59 @@ def create_app( app = FastAPI(title="OCRKit Model Studio", docs_url=None, redoc_url=None) + def _release_config() -> tuple[str, str, str]: + bucket = os.environ.get("OCRKIT_R2_DEFAULT_BUCKET", "").strip() + endpoint = os.environ.get("OCRKIT_R2_ENDPOINT_URL", "").strip() + access_key = os.environ.get("OCRKIT_R2_ACCESS_KEY_ID", "").strip() + secret_key = os.environ.get("OCRKIT_R2_SECRET_ACCESS_KEY", "").strip() + candidate_channel = os.environ.get( + "OCRKIT_MODEL_CANDIDATE_CHANNEL_KEY", "models/pp-ocrv6-small/channels/candidate.json" + ).strip() + stable_channel = os.environ.get( + "OCRKIT_MODEL_RELEASE_CHANNEL_KEY", "models/pp-ocrv6-small/channels/stable.json" + ).strip() + if not bucket or not endpoint or not access_key or not secret_key: + raise HTTPException(status_code=503, detail="模型发布未配置完整 R2 凭据") + return bucket, candidate_channel, stable_channel + + def _run_release_action(script_name: str, *extra: str) -> dict[str, object]: + bucket, candidate_channel, stable_channel = _release_config() + command = [ + os.fspath(ROOT / ".venv/bin/python") if (ROOT / ".venv/bin/python").is_file() else sys.executable, + str(ROOT / "training/scripts" / script_name), + "--bucket", bucket, + "--candidate-channel", candidate_channel, + "--stable-channel", stable_channel, + *extra, + ] + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, check=False) + if result.returncode != 0: + detail = result.stderr.strip() or result.stdout.strip() or f"{script_name} failed" + raise HTTPException(status_code=422, detail=detail) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise HTTPException(status_code=502, detail=f"{script_name} returned invalid JSON") from exc + if not isinstance(payload, dict): + raise HTTPException(status_code=502, detail=f"{script_name} returned an invalid response") + return payload + + @app.get("/api/model-release/compare") + def compare_model_release() -> dict[str, object]: + return _run_release_action("compare_model_channels.py") + + @app.post("/api/model-release/promote") + def promote_model_release(request: ReleaseAction) -> dict[str, object]: + if not request.confirmed: + raise HTTPException(status_code=422, detail="确认候选证据后才能晋级 stable") + return _run_release_action("promote_model_channel.py") + + @app.post("/api/model-release/rollback") + def rollback_model_release(request: RollbackAction) -> dict[str, object]: + if not request.confirmed: + raise HTTPException(status_code=422, detail="确认历史 manifest 后才能回滚 stable") + return _run_release_action("rollback_model_channel.py", "--manifest-key", request.manifest_key) + @app.get("/api/health") def health() -> dict[str, str]: return {"ok": "true", "service": "ocrkit-studio"} @@ -758,7 +821,12 @@ def publish(batch_id: str, request: PublishStart) -> dict[str, object]: run_dir = publication_root / datetime.now(UTC).strftime("release-%Y%m%d-%H%M%S") run_dir.mkdir(parents=True) log_path = run_dir / "release.log" - command = [str(ROOT / "training/release_rec_model.sh"), "--checkpoint", str(checkpoint)] + command = [str(ROOT / "training/release_rec_model.sh")] + command.extend([ + "--holdout-labels", str(batch_dir / "dataset/labels/holdout.txt"), + "--holdout-images-root", str(batch_dir / "dataset"), + "--provenance", str(batch_dir / "batch.json"), "--checkpoint", str(checkpoint), + ]) with log_path.open("ab") as log: process = subprocess.Popen(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) state: dict[str, object] = {"pid": process.pid, "status": "publishing", "command": command, "log": str(log_path), "checkpoint": str(checkpoint)} @@ -902,7 +970,7 @@ def snapshot_resume_checkpoints(ref: str) -> list[dict[str, str]]: def publish_snapshot(ref: str, request: PublishStart) -> dict[str, object]: if not request.confirmed: raise HTTPException(status_code=422, detail="confirm publication before writing model artifacts to R2") - _snapshot_dir(snapshot_import_root, ref) + import_dir = _snapshot_dir(snapshot_import_root, ref) run_root = _snapshot_run_root(work_root, ref) checkpoint = _checkpoint_from_training_state(run_root) publication_root = run_root / "publication" @@ -914,7 +982,12 @@ def publish_snapshot(ref: str, request: PublishStart) -> dict[str, object]: run_dir = publication_root / datetime.now(UTC).strftime("release-%Y%m%d-%H%M%S") run_dir.mkdir(parents=True) log_path = run_dir / "release.log" - command = [str(ROOT / "training/release_rec_model.sh"), "--checkpoint", str(checkpoint)] + command = [str(ROOT / "training/release_rec_model.sh")] + command.extend([ + "--holdout-labels", str(import_dir / "labels/holdout.txt"), + "--holdout-images-root", str(import_dir), + "--provenance", str(import_dir / "provenance.json"), "--checkpoint", str(checkpoint), + ]) with log_path.open("ab") as log: process = subprocess.Popen(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) state: dict[str, object] = {"pid": process.pid, "status": "publishing", "command": command, "log": str(log_path), "checkpoint": str(checkpoint)} diff --git a/training/studio/frontend/src/App.svelte b/training/studio/frontend/src/App.svelte index d6e3674..f050eca 100644 --- a/training/studio/frontend/src/App.svelte +++ b/training/studio/frontend/src/App.svelte @@ -12,6 +12,14 @@ log_tail?: string command?: string[] } + type ReleaseHistoryEntry = { manifest_key?: string; action?: string; verified_at?: string | null } + type ReleaseComparison = { + eligible: boolean + reasons: string[] + candidate: { version?: string; manifest_key?: string; channel_key?: string } + stable: { version?: string; manifest_key?: string; channel_key?: string; history?: ReleaseHistoryEntry[] } + comparison?: { field_accuracy?: { candidate?: number | null; stable?: number | null; delta?: number | null }; run_code_accuracy?: { candidate?: number | null; stable?: number | null; delta?: number | null } } + } type ResumeCheckpoint = { path: string; name: string } type SnapshotLabelConflict = { crop: string; transcriptions: string[]; annotation_ids: string[] } type SnapshotSummary = { @@ -179,6 +187,10 @@ let active = 'import' let training: TrainingState | null = null let publication: TrainingState | null = null + let releaseComparison: ReleaseComparison | null = null + let releaseConfirmed = false + let rollbackManifest = '' + let releaseBusy = false let publishConfirmed = false let resumeCheckpoints: ResumeCheckpoint[] = [] let resumeCheckpoint = '' @@ -1124,7 +1136,7 @@ } async function loadTrainingStep() { - await Promise.all([refreshTraining(), refreshResumeCheckpoints(), refreshPublication()]) + await Promise.all([refreshTraining(), refreshResumeCheckpoints(), refreshPublication(), refreshReleaseComparison({ silent: true })]) if (trainingIsRunning(training?.status)) startTrainingPoll() } @@ -1138,6 +1150,7 @@ // Surface completion even during silent polling so operators see the outcome. if (previous === 'publishing' && publication.status && publication.status !== 'publishing') { message(publication.status === 'completed' ? '模型已发布到 R2。' : '模型发布已结束，请查看发布日志。') + void refreshReleaseComparison({ silent: true }) } if (followPublishLog) await scrollLogToBottom() } catch (cause) { @@ -1145,6 +1158,38 @@ } } + async function refreshReleaseComparison(options?: { silent?: boolean }) { + try { + releaseComparison = await request('/api/model-release/compare') + const history = releaseComparison.stable.history || [] + if (rollbackManifest && !history.some((entry) => entry.manifest_key === rollbackManifest)) rollbackManifest = '' + } catch (cause) { + if (!options?.silent) message(cause instanceof Error ? cause.message : '获取模型比较结果失败', true) + } + } + + async function promoteStable() { + if (!releaseConfirmed) return message('请确认候选证据后再晋级 stable。', true) + releaseBusy = true + try { + await request('/api/model-release/promote', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ confirmed: true }) }) + releaseConfirmed = false + message('候选已显式晋级 stable；生产容器需按现有部署流程重启。') + await refreshReleaseComparison() + } catch (cause) { message(cause instanceof Error ? cause.message : '模型晋级失败', true) } finally { releaseBusy = false } + } + + async function rollbackStable() { + if (!rollbackManifest) return message('请选择一个历史 verified manifest。', true) + releaseBusy = true + try { + await request('/api/model-release/rollback', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ confirmed: true, manifest_key: rollbackManifest }) }) + message('stable 已回指历史 verified manifest；生产容器需按现有部署流程重启。') + rollbackManifest = '' + await refreshReleaseComparison() + } catch (cause) { message(cause instanceof Error ? cause.message : '模型回滚失败', true) } finally { releaseBusy = false } + } + async function startPublication() { if (!batch || !publishConfirmed) return message('请确认已准备将新模型写入 R2。', true) busy = true @@ -1208,7 +1253,7 @@ async function loadSnapshotStep() { if (!snapshot) return snapshotDetail = await request(`/api/snapshots/${encodeURIComponent(snapshot.ref)}`) - await Promise.all([refreshSnapshotTraining({ silent: true }), refreshSnapshotCheckpoints(), refreshSnapshotPublication({ silent: true })]) + await Promise.all([refreshSnapshotTraining({ silent: true }), refreshSnapshotCheckpoints(), refreshSnapshotPublication({ silent: true }), refreshReleaseComparison({ silent: true })]) if (snapshotTrainingIsRunning(snapshotTraining?.status)) startSnapshotPoll() } @@ -1279,6 +1324,7 @@ if (snapshotPublication.status !== 'publishing' && !snapshotTrainingIsRunning(snapshotTraining?.status)) stopSnapshotPoll() if (previous === 'publishing' && snapshotPublication.status && snapshotPublication.status !== 'publishing') { message(snapshotPublication.status === 'completed' ? '模型已发布到 R2。' : '模型发布已结束，请查看发布日志。') + void refreshReleaseComparison({ silent: true }) } if (snapshotFollowPublishLog) await scrollSnapshotLogs() } catch (cause) { @@ -2509,6 +2555,55 @@ {/if} +
+
+
+

模型决策

+

候选 → stable

+

候选发布只写入 candidate channel；只有比较报告和远端 artifact 校验都通过后，显式操作才会更新 stable。

+
+
+ +
+
+ {#if !releaseComparison} +

尚未读取模型 channel；配置 R2 后刷新比较。

+ {:else} +
+ 候选{releaseComparison.candidate.version || '未知'} · {releaseComparison.candidate.manifest_key || '无 manifest'} + stable{releaseComparison.stable.version || '未知'} · {releaseComparison.stable.manifest_key || '无 manifest'} +
+ {#if releaseComparison.comparison?.field_accuracy} +

fixture field accuracy：候选 {releaseComparison.comparison.field_accuracy.candidate ?? '—'}，stable {releaseComparison.comparison.field_accuracy.stable ?? '—'}，delta {releaseComparison.comparison.field_accuracy.delta ?? '—'}。

+ {/if} + {#if releaseComparison.eligible} +

候选证据完整，可执行显式晋级。

+ {:else} +
    + {#each releaseComparison.reasons as reason}
  • {reason}
    • {/each} +
+ {/if} +
+ + +
+
+ + +
+ {/if} +