Phase 2C.2: evaluate lightweight teaching-image routes

apps/api/tests/test_phase2c2_model_evaluation.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import struct
+import zlib
+from pathlib import Path
+
+from hcs_api.phase2c2_model_evaluation import (
+    EvaluationRunner,
+    aggregate_report,
+    build_tasks,
+    load_spec,
+    sha256_json,
+    verify_png,
+    write_review_package,
+)
+
+ROOT = Path(__file__).parents[3]
+SPEC = ROOT / "benchmarks/phase2c2/model-evaluation.v1.json"
+
+
+def _png(width: int = 512, height: int = 512) -> bytes:
+    raw = b"".join(b"\x00" + b"\x80\x90\xa0" * width for _ in range(height))
+    compressed = zlib.compress(raw)
+
+    def chunk(name: bytes, data: bytes) -> bytes:
+        return struct.pack(">I", len(data)) + name + data + struct.pack(">I", zlib.crc32(name + data) & 0xFFFFFFFF)
+
+    return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", compressed) + chunk(b"IEND", b"")
+
+
+def test_spec_has_six_cases_and_sana_is_fail_closed() -> None:
+    spec = load_spec(SPEC)
+    assert len(spec["cases"]) == 6
+    assert len(build_tasks(spec)) == 24
+    sana = next(item for item in spec["candidates"] if item["id"] == "sana-600m-512")
+    assert sana["run_enabled"] is False
+    assert "NVIDIA Processors" in sana["blocked_reason"]
+
+
+def test_png_verifier_checks_fixed_dimensions() -> None:
+    payload = _png()
+    verified = verify_png(payload, expected_width=512, expected_height=512)
+    assert verified["size_bytes"] == len(payload)
+    assert len(verified["sha256"]) == 64
+
+
+def test_runner_is_idempotent_and_writes_asset_manifest_and_reviews(tmp_path: Path) -> None:
+    spec = load_spec(SPEC)
+    spec["cases"] = spec["cases"][:1]
+    spec["candidates"] = [spec["candidates"][0]]
+    output = tmp_path / "run"
+    state_path = output / "state.json"
+    identity = {"runtime": "test", "model": "ssd-test"}
+    calls: list[str] = []
+
+    def execute(task):
+        calls.append(task.key)
+        return {"png_bytes": _png(), "technical": {"duration_seconds": 0.1}}
+
+    runner = EvaluationRunner(spec, output, state_path, identity)
+    state = runner.run(execute)
+    assert state["status"] == "completed"
+    assert len(calls) == 4
+    runner = EvaluationRunner(spec, output, state_path, identity)
+    runner.run(execute)
+    assert len(calls) == 4
+    assert len(__import__("json").loads((output / "asset_manifest.json").read_text())["images"]) == 4
+    package = write_review_package(spec, runner.state, output)
+    assert (package / "comparison.html").is_file()
+    assert len(__import__("json").loads((package / "teacher-reviews.pending.json").read_text())["reviews"]) == 4
+
+
+def test_runner_continues_after_failure_and_retries(tmp_path: Path) -> None:
+    spec = load_spec(SPEC)
+    spec["cases"] = spec["cases"][:1]
+    spec["candidates"] = [spec["candidates"][0]]
+    output = tmp_path / "run"
+    state_path = output / "state.json"
+    identity = {"runtime": "test", "model": "ssd-test"}
+    failed_once = {build_tasks(spec)[0].key}
+
+    def execute(task):
+        if task.key in failed_once:
+            failed_once.remove(task.key)
+            raise RuntimeError("one controlled failure")
+        return {"png_bytes": _png(), "technical": {"duration_seconds": 0.1}}
+
+    runner = EvaluationRunner(spec, output, state_path, identity)
+    state = runner.run(execute, max_attempts=1)
+    assert state["status"] == "completed_with_failures"
+    assert sum(item["status"] == "succeeded" for item in state["tasks"].values()) == 3
+    runner = EvaluationRunner(spec, output, state_path, identity)
+    state = runner.run(execute, max_attempts=2)
+    assert state["status"] == "completed"
+    report = aggregate_report(spec, state, output)
+    assert report["failed"] == 0
+    variant_report = report["candidate_results"]["ssd-1b"]["variants"]["base"]
+    assert variant_report["technical_success_rate"] == 1.0
+    assert variant_report["mean_duration_seconds"] == 0.1
+    assert sha256_json(identity) == state["identity_sha256"]

benchmarks/phase2c2/candidate-audit.v1.json

@@ -0,0 +1,71 @@
+{
+  "schema": "hanclassstudio.phase2c2_candidate_audit.v1",
+  "version": "1.0.0",
+  "host_contract": {
+    "operating_system": "macos",
+    "architecture": "arm64",
+    "memory_bytes": 17179869184,
+    "mps_available": true,
+    "audit_date": "2026-07-28"
+  },
+  "candidates": [
+    {
+      "id": "ssd-1b",
+      "role": "formal_product_candidate",
+      "repository": "https://huggingface.co/segmind/SSD-1B",
+      "revision": "60987f37e94cd59c36b1cba832b9f97b57395a10",
+      "model_card_license": "Apache-2.0",
+      "upstream_source": "Segmind official Hugging Face repository",
+      "architecture": "SDXL distilled UNet pipeline, approximately 1.3B parameters",
+      "selected_precision": "fp16 safetensors components",
+      "selected_download_bytes": 4465653694,
+      "selected_files": [
+        {"path": "text_encoder/model.fp16.safetensors", "bytes": 246144864, "sha256": "5487ea0eee9c9a9bff8abd097908d4deff3ae1fa87b3b67397f8b9538139d447"},
+        {"path": "text_encoder_2/model.fp16.safetensors", "bytes": 1389382880, "sha256": "d3df577f6e3799c8e1bd9b40e30133710e02e8e25d0ce48cdcc790e7dfe12d6d"},
+        {"path": "unet/diffusion_pytorch_model.fp16.safetensors", "bytes": 2662790608, "sha256": "40d8ea9159f3e875278dacc7879442d58c45850cf13c62f5e26681061c51829a"},
+        {"path": "vae/diffusion_pytorch_model.fp16.safetensors", "bytes": 167335342, "sha256": "6353737672c94b96174cb590f711eac6edf2fcce5b6e91aa9d73c5adc589ee48"}
+      ],
+      "framework": "Hugging Face Diffusers StableDiffusionXLPipeline; model card also claims ComfyUI compatibility",
+      "apple_silicon": "not promised by the model card; tested opt-in on this MPS host",
+      "lora": "official Diffusers train_text_to_image_lora_sdxl.py path is documented",
+      "custom_nodes": false,
+      "supply_chain_notes": [
+        "Model card lists GRIT and a Midjourney scrape as training data.",
+        "Apache-2.0 covers the published model repository; training-data provenance is a separate product review risk."
+      ],
+      "status": "audit_pass_download_and_real_pilot"
+    },
+    {
+      "id": "sana-600m-512",
+      "role": "research_candidate",
+      "repository": "https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers",
+      "revision": "2defc07f5fb66d0c53ace051585e9a2cb83f8c15",
+      "model_card_license": "NSCL v2-custom / NVIDIA License",
+      "license_source": "https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers/blob/2defc07f5fb66d0c53ace051585e9a2cb83f8c15/LICENSE.txt",
+      "license_constraints": [
+        "non-commercial research or evaluation only",
+        "use only with NVIDIA Processors",
+        "NSFW filtering through the separately obtained Safe Model is required"
+      ],
+      "architecture": "Sana 0.6B linear diffusion transformer with Gemma2 2B text encoder and 32x compressed VAE",
+      "selected_precision": "fp16 safetensors components",
+      "selected_download_bytes": 7699969740,
+      "selected_files": [
+        {"path": "text_encoder/model.fp16-00001-of-00002.safetensors", "bytes": 4988024144, "sha256": "2ec6c12cdb4f33eeb8fc0e96e081187943a90a70"},
+        {"path": "text_encoder/model.fp16-00002-of-00002.safetensors", "bytes": 240691624, "sha256": "56158202c6d7382ec0e4235466b897fba141aa52"},
+        {"path": "transformer/diffusion_pytorch_model.fp16.safetensors", "bytes": 1183558264, "sha256": "4616f5f26161c4f072083edcd9b771b8a840057602e928dca00c8a05175b22a3"},
+        {"path": "vae/diffusion_pytorch_model.fp16.safetensors", "bytes": 1249044836, "sha256": "15a4b09e56d95b768a0ec9da50b702e21d920333fc9b3480d66bb5c7fad9d87f"}
+      ],
+      "framework": "official Sana repository or Diffusers SanaPipeline; official ComfyUI guide requires ComfyUI_ExtraModels custom nodes and a custom VAE",
+      "apple_silicon": "not permitted by the NVIDIA License; no download, training, or inference on this host",
+      "lora": "official Diffusers Sana LoRA/DreamBooth training path exists, but cannot be used under the current host/license contract",
+      "custom_nodes": true,
+      "status": "fail_closed_license_and_platform"
+    }
+  ],
+  "baseline_reference": {
+    "model": "hcs.sd15-teaching-illustration-fp16",
+    "source": "Phase 2C.1 SD 1.5 ablation report in the prior ignored runtime worktree",
+    "not_rerun_in_this_loop": true
+  }
+}