From 0320847677eb14aaadb82747d93d8dc40e4cf3d1 Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:03:38 +0700 Subject: [PATCH 1/3] feat(phase2c2): add controlled model evaluation contract and dataset --- .../src/hcs_api/phase2c2_model_evaluation.py | 546 ++++++++++++++++++ .../tests/test_phase2c2_model_evaluation.py | 98 ++++ benchmarks/phase2c2/candidate-audit.v1.json | 71 +++ .../phase2c2/generate_flat_cartoon_dataset.py | 229 ++++++++ benchmarks/phase2c2/model-evaluation.v1.json | 136 +++++ benchmarks/phase2c2/run_model_evaluation.py | 194 +++++++ .../phase2c2/train_flat_cartoon_lora.py | 76 +++ 7 files changed, 1350 insertions(+) create mode 100644 apps/api/src/hcs_api/phase2c2_model_evaluation.py create mode 100644 apps/api/tests/test_phase2c2_model_evaluation.py create mode 100644 benchmarks/phase2c2/candidate-audit.v1.json create mode 100644 benchmarks/phase2c2/generate_flat_cartoon_dataset.py create mode 100644 benchmarks/phase2c2/model-evaluation.v1.json create mode 100644 benchmarks/phase2c2/run_model_evaluation.py create mode 100644 benchmarks/phase2c2/train_flat_cartoon_lora.py diff --git a/apps/api/src/hcs_api/phase2c2_model_evaluation.py b/apps/api/src/hcs_api/phase2c2_model_evaluation.py new file mode 100644 index 0000000..ae2384f --- /dev/null +++ b/apps/api/src/hcs_api/phase2c2_model_evaluation.py @@ -0,0 +1,546 @@ +"""Small, resumable model-evaluation contract for Phase 2C.2. + +This module is deliberately backend-only and provider-neutral. It records the +same request/artifact/provenance/Asset Manifest relationships as the teaching +image path, while the actual Diffusers executor lives in the ignored runtime +directory and never changes production model or workflow defaults. +""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import shutil +import struct +import time +import uuid +import zlib +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Iterable + + +SCHEMA = "hanclassstudio.phase2c2_model_evaluation.v1" +REVIEW_SCHEMA = "hanclassstudio.phase2c2_model_evaluation_review.v1" +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +_MAX_PNG_BYTES = 32 * 1024 * 1024 + + +class ModelEvaluationError(RuntimeError): + pass + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_json(value: Any) -> str: + return sha256_bytes(canonical(value)) + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ModelEvaluationError(f"invalid JSON: {path}") from exc + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_bytes(canonical(value) + b"\n") + os.replace(tmp, path) + + +def _safe_name(value: str) -> str: + if not value or any(ch not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-" for ch in value): + raise ModelEvaluationError(f"unsafe output name: {value!r}") + return value + + +@dataclass(frozen=True) +class EvaluationTask: + key: str + candidate_id: str + variant_id: str + case_id: str + seed: int + request: dict[str, Any] + plan: dict[str, Any] + + +def load_spec(path: Path) -> dict[str, Any]: + spec = read_json(path) + if not isinstance(spec, dict) or spec.get("schema") != SCHEMA: + raise ModelEvaluationError("model-evaluation spec schema mismatch") + if not isinstance(spec.get("candidates"), list) or not spec["candidates"]: + raise ModelEvaluationError("model-evaluation spec has no candidates") + if not isinstance(spec.get("cases"), list) or not spec["cases"]: + raise ModelEvaluationError("model-evaluation spec has no cases") + if spec.get("dimensions", {}).get("width", 0) <= 0 or spec.get("dimensions", {}).get("height", 0) <= 0: + raise ModelEvaluationError("model-evaluation dimensions are invalid") + for case in spec["cases"]: + if not isinstance(case, dict) or not case.get("id") or not case.get("prompt"): + raise ModelEvaluationError("each case needs an id and prompt") + if not case.get("seeds"): + raise ModelEvaluationError(f"case {case.get('id')} has no fixed seeds") + return spec + + +def _candidate(spec: dict[str, Any], candidate_id: str) -> dict[str, Any]: + try: + return next(item for item in spec["candidates"] if item["id"] == candidate_id) + except StopIteration as exc: + raise ModelEvaluationError(f"unknown candidate: {candidate_id}") from exc + + +def _case(spec: dict[str, Any], case_id: str) -> dict[str, Any]: + try: + return next(item for item in spec["cases"] if item["id"] == case_id) + except StopIteration as exc: + raise ModelEvaluationError(f"unknown case: {case_id}") from exc + + +def _variant(spec: dict[str, Any], variant_id: str) -> dict[str, Any]: + try: + return next(item for item in spec["variants"] if item["id"] == variant_id) + except StopIteration as exc: + raise ModelEvaluationError(f"unknown variant: {variant_id}") from exc + + +def _request_for(spec: dict[str, Any], candidate_id: str, variant_id: str, case: dict[str, Any], seed: int) -> tuple[dict[str, Any], dict[str, Any]]: + candidate = _candidate(spec, candidate_id) + variant = _variant(spec, variant_id) + task_id = f"{candidate_id}:{variant_id}:{case['id']}:{seed}" + task_hash = sha256_bytes(task_id.encode("utf-8")) + asset_id = f"eval-{candidate_id}-{variant_id}-{case['id']}-{seed}"[:78] + prompt_profile = variant["prompt_profile"] + request = { + "schema": "hanclassstudio.teaching_image_request.v1", + "asset_id": asset_id, + "purpose": "teaching_illustration", + "subject": case["expected"], + "action": case["prompt"], + "environment": case["environment"], + "aspect_ratio": "1:1", + "seed": seed, + "source_trace": [ + "benchmark:phase2c2-model-evaluation", + f"candidate:{candidate_id}", + f"variant:{variant_id}", + f"case:{case['id']}", + f"prompt-profile:{prompt_profile['id']}", + f"seed:{seed}", + ], + } + request_sha = sha256_json(request) + positive = f"{prompt_profile['positive_prefix']}, {case['prompt']}" + negative = prompt_profile["negative"] + plan = { + "schema": "hanclassstudio.phase2c2_execution_plan.v1", + "task_id": task_id, + "request_sha256": request_sha, + "candidate_id": candidate_id, + "model_revision": candidate["revision"], + "variant_id": variant_id, + "prompt_profile_id": prompt_profile["id"], + "positive_prompt": positive, + "negative_prompt": negative, + "width": spec["dimensions"]["width"], + "height": spec["dimensions"]["height"], + "seed": seed, + "steps": variant["steps"], + "guidance_scale": variant["guidance_scale"], + "scheduler": variant["scheduler"], + "sampler": variant["sampler"], + "source_trace": request["source_trace"], + } + plan["execution_plan_sha256"] = sha256_json(plan) + return request, plan + + +def build_tasks(spec: dict[str, Any], candidate_ids: Iterable[str] | None = None) -> list[EvaluationTask]: + allowed = set(candidate_ids) if candidate_ids is not None else None + tasks: list[EvaluationTask] = [] + for candidate in spec["candidates"]: + candidate_id = candidate["id"] + if allowed is not None and candidate_id not in allowed: + continue + if not candidate.get("run_enabled", False): + continue + for variant in spec["variants"]: + for case in spec["cases"]: + for seed in case["seeds"]: + request, plan = _request_for(spec, candidate_id, variant["id"], case, int(seed)) + tasks.append(EvaluationTask( + key=plan["task_id"], candidate_id=candidate_id, variant_id=variant["id"], + case_id=case["id"], seed=int(seed), request=request, plan=plan, + )) + return tasks + + +def verify_png(payload: bytes, *, expected_width: int, expected_height: int) -> dict[str, Any]: + if len(payload) <= 33 or len(payload) > _MAX_PNG_BYTES or not payload.startswith(_PNG_SIGNATURE): + raise ModelEvaluationError("PNG signature or size is invalid") + offset = len(_PNG_SIGNATURE) + seen_ihdr = False + seen_idat = False + seen_iend = False + chunks = 0 + while offset + 12 <= len(payload): + length = struct.unpack(">I", payload[offset : offset + 4])[0] + chunk_start = offset + 8 + chunk_end = chunk_start + length + crc_end = chunk_end + 4 + if chunk_end > len(payload) or crc_end > len(payload): + raise ModelEvaluationError("PNG chunk exceeds payload") + name = payload[offset + 4 : offset + 8] + data = payload[chunk_start:chunk_end] + expected_crc = struct.unpack(">I", payload[chunk_end:crc_end])[0] + if zlib.crc32(name + data) & 0xFFFFFFFF != expected_crc: + raise ModelEvaluationError("PNG CRC mismatch") + chunks += 1 + if name == b"IHDR": + if seen_ihdr or len(data) != 13: + raise ModelEvaluationError("PNG IHDR is invalid") + seen_ihdr = True + width, height = struct.unpack(">II", data[:8]) + if width != expected_width or height != expected_height: + raise ModelEvaluationError("PNG dimensions differ from the fixed experiment") + elif name == b"IDAT": + seen_idat = True + elif name == b"IEND": + seen_iend = True + if crc_end != len(payload): + raise ModelEvaluationError("PNG has trailing bytes") + break + offset = crc_end + if not (seen_ihdr and seen_idat and seen_iend): + raise ModelEvaluationError("PNG is missing a required chunk") + return {"width": expected_width, "height": expected_height, "size_bytes": len(payload), "sha256": sha256_bytes(payload), "chunk_count": chunks} + + +def _new_state(spec: dict[str, Any], tasks: list[EvaluationTask], identity: dict[str, Any]) -> dict[str, Any]: + return { + "schema": "hanclassstudio.phase2c2_model_evaluation_state.v1", + "experiment_id": spec["experiment_id"], + "spec_sha256": sha256_json(spec), + "identity": identity, + "identity_sha256": sha256_json(identity), + "status": "pending", + "created_at": utc_now(), + "updated_at": utc_now(), + "tasks": { + task.key: { + "task": task.key, + "candidate_id": task.candidate_id, + "variant_id": task.variant_id, + "case_id": task.case_id, + "seed": task.seed, + "request": task.request, + "plan": task.plan, + "status": "pending", + "attempts": 0, + "result": None, + "failure": None, + } + for task in tasks + }, + } + + +def _atomic_write(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_bytes(data) + os.replace(tmp, path) + + +class EvaluationRunner: + """Stateful runner; one state write occurs after every task attempt.""" + + def __init__(self, spec: dict[str, Any], output_dir: Path, state_path: Path, identity: dict[str, Any]) -> None: + self.spec = spec + self.output_dir = output_dir + self.state_path = state_path + self.identity = identity + self.tasks = build_tasks(spec) + self.state = self._load_or_create() + + def _load_or_create(self) -> dict[str, Any]: + if not self.state_path.exists(): + state = _new_state(self.spec, self.tasks, self.identity) + write_json(self.state_path, state) + return state + state = read_json(self.state_path) + if state.get("spec_sha256") != sha256_json(self.spec): + raise ModelEvaluationError("state belongs to a different experiment spec") + current_identity = sha256_json(self.identity) + if state.get("identity_sha256") != current_identity: + invalidated = dict(state) + invalidated["status"] = "invalidated" + invalidated["invalidated_at"] = utc_now() + invalidated["invalidated_reason"] = "runtime/model/pipeline identity changed" + write_json(self.state_path.with_suffix(".invalidated.json"), invalidated) + state = _new_state(self.spec, self.tasks, self.identity) + state["invalidated_previous_state"] = True + write_json(self.state_path, state) + return state + + def _save(self) -> None: + self.state["updated_at"] = utc_now() + write_json(self.state_path, self.state) + + def _artifact_valid(self, record: dict[str, Any]) -> bool: + result = record.get("result") or {} + image = self.output_dir / result.get("image_path", "") + provenance = self.output_dir / result.get("provenance_path", "") + if not image.is_file() or not provenance.is_file(): + return False + try: + image_bytes = image.read_bytes() + if sha256_bytes(image_bytes) != result.get("sha256"): + return False + if sha256_bytes(provenance.read_bytes()) != result.get("provenance_sha256"): + return False + return result.get("identity_sha256") == sha256_json(self.identity) + except OSError: + return False + + def _persist(self, task: EvaluationTask, rendered: dict[str, Any]) -> dict[str, Any]: + png = rendered.get("png_bytes") + if not isinstance(png, bytes): + raise ModelEvaluationError("executor did not return PNG bytes") + tech = verify_png(png, expected_width=self.spec["dimensions"]["width"], expected_height=self.spec["dimensions"]["height"]) + task_dir = self.output_dir / "images" / _safe_name(task.candidate_id) / _safe_name(task.variant_id) + provenance_dir = self.output_dir / "provenance" + image_name = f"{_safe_name(task.case_id)}-{task.seed}.png" + provenance_name = f"{_safe_name(task.candidate_id)}-{_safe_name(task.variant_id)}-{_safe_name(task.case_id)}-{task.seed}.json" + image_path = task_dir / image_name + provenance_path = provenance_dir / provenance_name + provenance = { + "schema": "hanclassstudio.phase2c2_model_evaluation_provenance.v1", + "request": task.request, + "plan": task.plan, + "identity": self.identity, + "technical": {**tech, **(rendered.get("technical") or {})}, + "source_trace": task.request["source_trace"], + "generated_at": utc_now(), + } + provenance_bytes = canonical(provenance) + b"\n" + _atomic_write(image_path, png) + _atomic_write(provenance_path, provenance_bytes) + artifact_seed = f"{task.key}:{tech['sha256']}".encode() + artifact_id = f"img-{sha256_bytes(artifact_seed)[:24]}" + relative_image = image_path.relative_to(self.output_dir).as_posix() + relative_provenance = provenance_path.relative_to(self.output_dir).as_posix() + result = { + "task": task.key, + "request_sha256": task.plan["request_sha256"], + "execution_plan_sha256": task.plan["execution_plan_sha256"], + "artifact_id": artifact_id, + "asset_id": task.request["asset_id"], + "image_path": relative_image, + "provenance_path": relative_provenance, + "sha256": tech["sha256"], + "size_bytes": tech["size_bytes"], + "width": tech["width"], + "height": tech["height"], + "provenance_sha256": sha256_bytes(provenance_bytes), + "identity_sha256": sha256_json(self.identity), + "technical": provenance["technical"], + "review_status": "pending_review", + "failure_tags": [], + } + self._write_asset_manifest(result, task, provenance) + return result + + def _write_asset_manifest(self, result: dict[str, Any], task: EvaluationTask, provenance: dict[str, Any]) -> None: + manifest_path = self.output_dir / "asset_manifest.json" + manifest = read_json(manifest_path) if manifest_path.exists() else { + "schema": "hanclassstudio.asset_manifest.v1", + "images": [], + } + images = [item for item in manifest.get("images", []) if item.get("id") != result["asset_id"]] + images.append({ + "id": result["asset_id"], + "kind": "image", + "path": result["image_path"], + "placeholder": False, + "mime_type": "image/png", + "content_hash": result["sha256"], + "review_state": "pending_review", + "request_fingerprint": result["request_sha256"], + "generation": { + "provider": "hcs.phase2c2.diffusers-evaluation", + "model": task.candidate_id, + "local_path": result["image_path"], + "mime_type": "image/png", + "width": result["width"], + "height": result["height"], + "prompt": task.plan["positive_prompt"], + "style_profile": task.plan["prompt_profile_id"], + "seed": task.seed, + "content_hash": result["sha256"], + "source_trace": task.request["source_trace"], + }, + "verified_image_artifact": { + "schema": "hanclassstudio.phase2c2_verified_image_artifact.v1", + "artifact_id": result["artifact_id"], + "asset_id": result["asset_id"], + "path": result["image_path"], + "mime_type": "image/png", + "width": result["width"], + "height": result["height"], + "size_bytes": result["size_bytes"], + "sha256": result["sha256"], + "provenance_ref": result["provenance_path"], + "provenance_sha256": result["provenance_sha256"], + "provenance": provenance, + }, + }) + manifest["images"] = images + write_json(manifest_path, manifest) + + def run(self, executor: Callable[[EvaluationTask], dict[str, Any]], *, max_attempts: int = 2, stop_after: int | None = None) -> dict[str, Any]: + attempted = 0 + self.state["status"] = "running" + self._save() + try: + for task in self.tasks: + record = self.state["tasks"][task.key] + if record["status"] == "succeeded" and self._artifact_valid(record): + continue + if record["status"] == "succeeded": + record["status"] = "pending" + record["result"] = None + while record["attempts"] < max_attempts: + record["attempts"] += 1 + attempted += 1 + try: + result = self._persist(task, executor(task)) + record["status"] = "succeeded" + record["result"] = result + record["failure"] = None + self._save() + break + except Exception as exc: # individual failure never aborts the batch + record["failure"] = {"type": type(exc).__name__, "message": str(exc), "at": utc_now()} + record["status"] = "failed" + self._save() + if stop_after is not None and attempted >= stop_after: + self.state["status"] = "paused" + self._save() + return self.state + except KeyboardInterrupt: + self.state["status"] = "paused" + self._save() + return self.state + self.state["status"] = "completed" if all(item["status"] == "succeeded" for item in self.state["tasks"].values()) else "completed_with_failures" + self._save() + return self.state + + +def write_review_package(spec: dict[str, Any], state: dict[str, Any], output_dir: Path) -> Path: + package = output_dir / "review-package" + images_dir = package / "images" + images_dir.mkdir(parents=True, exist_ok=True) + records = [record for record in state["tasks"].values() if record.get("status") == "succeeded" and record.get("result")] + for record in records: + source = output_dir / record["result"]["image_path"] + review_name = f"{record['candidate_id']}-{record['variant_id']}-{source.name}" + target = images_dir / review_name + if source.is_file(): + shutil.copy2(source, target) + fields = ["candidate_id", "variant_id", "case_id", "seed", "asset_id", "artifact_id", "review_status", "visual_quality", "prompt_adherence", "object_count_accuracy", "action_accuracy", "spatial_relation_accuracy", "teaching_usability", "anatomy_quality", "needs_regeneration", "failure_tags", "reviewer_notes"] + pending = [] + for record in records: + source_name = Path(record["result"]["image_path"]).name + review_name = f"{record['candidate_id']}-{record['variant_id']}-{source_name}" + task = record["task"] + pending.append({ + "schema": REVIEW_SCHEMA, + "task": task, + "candidate_id": record["candidate_id"], + "variant_id": record["variant_id"], + "case_id": record["case_id"], + "seed": record["seed"], + "asset_id": record["result"]["asset_id"], + "artifact_id": record["result"]["artifact_id"], + "image_path": f"images/{review_name}", + "review_status": "pending_review", + "visual_quality": None, + "prompt_adherence": None, + "object_count_accuracy": None, + "action_accuracy": None, + "spatial_relation_accuracy": None, + "teaching_usability": None, + "anatomy_quality": None, + "needs_regeneration": None, + "failure_tags": [], + "reviewer_notes": "", + }) + write_json(package / "teacher-reviews.pending.json", {"schema": REVIEW_SCHEMA, "reviews": pending}) + with (package / "teacher-reviews.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + for item in pending: + writer.writerow({field: json.dumps(item[field], ensure_ascii=False) if isinstance(item[field], list) else item[field] for field in fields}) + write_json(package / "experiment-manifest.json", {"schema": SCHEMA, "spec": spec, "results": records}) + grouped: dict[tuple[str, str, int], list[dict[str, Any]]] = {} + for record in records: + grouped.setdefault((record["case_id"], str(record["seed"]), record["candidate_id"]), []).append(record) + cards: list[str] = [] + for record in records: + result = record["result"] + review_name = f"{record['candidate_id']}-{record['variant_id']}-{Path(result['image_path']).name}" + cards.append( + f'

{record["candidate_id"]} / {record["variant_id"]} / {record["case_id"]} / seed {record["seed"]}

' + f'pending teacher review' + f'

artifact {result["artifact_id"]} · technical {result["technical"].get("width")}×{result["technical"].get("height")}

' + ) + html = "Phase 2C.2 model evaluation

Phase 2C.2 model evaluation

Technical checks are separate from teacher review. All review fields are pending.

" + "".join(cards) + "
\n" + (package / "comparison.html").write_text(html, encoding="utf-8") + (package / "README.md").write_text("# Phase 2C.2 evaluation review package\n\nOpen `comparison.html`. Technical checks are automatic; teacher fields remain `pending_review`.\n", encoding="utf-8") + return package + + +def aggregate_report(spec: dict[str, Any], state: dict[str, Any], output_dir: Path, *, blockers: list[dict[str, Any]] | None = None) -> dict[str, Any]: + records = list(state["tasks"].values()) + report: dict[str, Any] = { + "schema": "hanclassstudio.phase2c2_model_evaluation_report.v1", + "experiment_id": spec["experiment_id"], + "spec_sha256": state["spec_sha256"], + "identity": state["identity"], + "run_status": state["status"], + "total": len(records), + "succeeded": sum(item["status"] == "succeeded" for item in records), + "failed": sum(item["status"] == "failed" for item in records), + "pending": sum(item["status"] not in {"succeeded", "failed"} for item in records), + "teacher_review": {"pending": sum(item["status"] == "succeeded" for item in records), "reviewed": 0, "automatic_scores": False, "conclusion": None}, + "blockers": blockers or [], + "candidate_results": {}, + "limitations": ["No teacher scores are inferred from technical or visual checks."], + } + for candidate in spec["candidates"]: + cid = candidate["id"] + if candidate.get("run_enabled", False): + subset = [item for item in records if item["candidate_id"] == cid] + report["candidate_results"][cid] = {"total": len(subset), "succeeded": sum(item["status"] == "succeeded" for item in subset), "failed": sum(item["status"] == "failed" for item in subset), "variants": {}} + for variant in spec["variants"]: + v = [item for item in subset if item["variant_id"] == variant["id"]] + durations = [item["result"]["technical"].get("duration_seconds") for item in v if item.get("result") and item["result"]["technical"].get("duration_seconds") is not None] + report["candidate_results"][cid]["variants"][variant["id"]] = {"total": len(v), "succeeded": sum(item["status"] == "succeeded" for item in v), "failed": sum(item["status"] == "failed" for item in v), "mean_duration_seconds": (sum(durations) / len(durations) if durations else None)} + write_json(output_dir / "model-evaluation-report.json", report) + return report diff --git a/apps/api/tests/test_phase2c2_model_evaluation.py b/apps/api/tests/test_phase2c2_model_evaluation.py new file mode 100644 index 0000000..2704bfa --- /dev/null +++ b/apps/api/tests/test_phase2c2_model_evaluation.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import struct +import zlib +from pathlib import Path + +from hcs_api.phase2c2_model_evaluation import ( + EvaluationRunner, + aggregate_report, + build_tasks, + load_spec, + sha256_json, + verify_png, + write_review_package, +) + + +ROOT = Path(__file__).parents[3] +SPEC = ROOT / "benchmarks/phase2c2/model-evaluation.v1.json" + + +def _png(width: int = 512, height: int = 512) -> bytes: + raw = b"".join(b"\x00" + b"\x80\x90\xa0" * width for _ in range(height)) + compressed = zlib.compress(raw) + + def chunk(name: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + name + data + struct.pack(">I", zlib.crc32(name + data) & 0xFFFFFFFF) + + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", compressed) + chunk(b"IEND", b"") + + +def test_spec_has_six_cases_and_sana_is_fail_closed() -> None: + spec = load_spec(SPEC) + assert len(spec["cases"]) == 6 + assert len(build_tasks(spec)) == 24 + sana = next(item for item in spec["candidates"] if item["id"] == "sana-600m-512") + assert sana["run_enabled"] is False + assert "NVIDIA Processors" in sana["blocked_reason"] + + +def test_png_verifier_checks_fixed_dimensions() -> None: + payload = _png() + verified = verify_png(payload, expected_width=512, expected_height=512) + assert verified["size_bytes"] == len(payload) + assert len(verified["sha256"]) == 64 + + +def test_runner_is_idempotent_and_writes_asset_manifest_and_reviews(tmp_path: Path) -> None: + spec = load_spec(SPEC) + spec["cases"] = spec["cases"][:1] + spec["candidates"] = [spec["candidates"][0]] + output = tmp_path / "run" + state_path = output / "state.json" + identity = {"runtime": "test", "model": "ssd-test"} + calls: list[str] = [] + + def execute(task): + calls.append(task.key) + return {"png_bytes": _png(), "technical": {"duration_seconds": 0.1}} + + runner = EvaluationRunner(spec, output, state_path, identity) + state = runner.run(execute) + assert state["status"] == "completed" + assert len(calls) == 4 + runner = EvaluationRunner(spec, output, state_path, identity) + runner.run(execute) + assert len(calls) == 4 + assert len(__import__("json").loads((output / "asset_manifest.json").read_text())["images"]) == 4 + package = write_review_package(spec, runner.state, output) + assert (package / "comparison.html").is_file() + assert len(__import__("json").loads((package / "teacher-reviews.pending.json").read_text())["reviews"]) == 4 + + +def test_runner_continues_after_failure_and_retries(tmp_path: Path) -> None: + spec = load_spec(SPEC) + spec["cases"] = spec["cases"][:1] + spec["candidates"] = [spec["candidates"][0]] + output = tmp_path / "run" + state_path = output / "state.json" + identity = {"runtime": "test", "model": "ssd-test"} + failed_once = {build_tasks(spec)[0].key} + + def execute(task): + if task.key in failed_once: + failed_once.remove(task.key) + raise RuntimeError("one controlled failure") + return {"png_bytes": _png(), "technical": {"duration_seconds": 0.1}} + + runner = EvaluationRunner(spec, output, state_path, identity) + state = runner.run(execute, max_attempts=1) + assert state["status"] == "completed_with_failures" + assert sum(item["status"] == "succeeded" for item in state["tasks"].values()) == 3 + runner = EvaluationRunner(spec, output, state_path, identity) + state = runner.run(execute, max_attempts=2) + assert state["status"] == "completed" + report = aggregate_report(spec, state, output) + assert report["failed"] == 0 + assert sha256_json(identity) == state["identity_sha256"] diff --git a/benchmarks/phase2c2/candidate-audit.v1.json b/benchmarks/phase2c2/candidate-audit.v1.json new file mode 100644 index 0000000..10ceea0 --- /dev/null +++ b/benchmarks/phase2c2/candidate-audit.v1.json @@ -0,0 +1,71 @@ +{ + "schema": "hanclassstudio.phase2c2_candidate_audit.v1", + "version": "1.0.0", + "host_contract": { + "operating_system": "macos", + "architecture": "arm64", + "memory_bytes": 17179869184, + "mps_available": true, + "audit_date": "2026-07-28" + }, + "candidates": [ + { + "id": "ssd-1b", + "role": "formal_product_candidate", + "repository": "https://huggingface.co/segmind/SSD-1B", + "revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", + "model_card_license": "Apache-2.0", + "upstream_source": "Segmind official Hugging Face repository", + "architecture": "SDXL distilled UNet pipeline, approximately 1.3B parameters", + "selected_precision": "fp16 safetensors components", + "selected_download_bytes": 4465653694, + "selected_files": [ + {"path": "text_encoder/model.fp16.safetensors", "bytes": 246144864, "sha256": "5487ea0eee9c9a9bff8abd097908d4deff3ae1fa87b3b67397f8b9538139d447"}, + {"path": "text_encoder_2/model.fp16.safetensors", "bytes": 1389382880, "sha256": "d3df577f6e3799c8e1bd9b40e30133710e02e8e25d0ce48cdcc790e7dfe12d6d"}, + {"path": "unet/diffusion_pytorch_model.fp16.safetensors", "bytes": 2662790608, "sha256": "40d8ea9159f3e875278dacc7879442d58c45850cf13c62f5e26681061c51829a"}, + {"path": "vae/diffusion_pytorch_model.fp16.safetensors", "bytes": 167335342, "sha256": "6353737672c94b96174cb590f711eac6edf2fcce5b6e91aa9d73c5adc589ee48"} + ], + "framework": "Hugging Face Diffusers StableDiffusionXLPipeline; model card also claims ComfyUI compatibility", + "apple_silicon": "not promised by the model card; tested opt-in on this MPS host", + "lora": "official Diffusers train_text_to_image_lora_sdxl.py path is documented", + "custom_nodes": false, + "supply_chain_notes": [ + "Model card lists GRIT and a Midjourney scrape as training data.", + "Apache-2.0 covers the published model repository; training-data provenance is a separate product review risk." + ], + "status": "audit_pass_download_and_real_pilot" + }, + { + "id": "sana-600m-512", + "role": "research_candidate", + "repository": "https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers", + "revision": "2defc07f5fb66d0c53ace051585e9a2cb83f8c15", + "model_card_license": "NSCL v2-custom / NVIDIA License", + "license_source": "https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers/blob/2defc07f5fb66d0c53ace051585e9a2cb83f8c15/LICENSE.txt", + "license_constraints": [ + "non-commercial research or evaluation only", + "use only with NVIDIA Processors", + "NSFW filtering through the separately obtained Safe Model is required" + ], + "architecture": "Sana 0.6B linear diffusion transformer with Gemma2 2B text encoder and 32x compressed VAE", + "selected_precision": "fp16 safetensors components", + "selected_download_bytes": 7699969740, + "selected_files": [ + {"path": "text_encoder/model.fp16-00001-of-00002.safetensors", "bytes": 4988024144, "sha256": "2ec6c12cdb4f33eeb8fc0e96e081187943a90a70"}, + {"path": "text_encoder/model.fp16-00002-of-00002.safetensors", "bytes": 240691624, "sha256": "56158202c6d7382ec0e4235466b897fba141aa52"}, + {"path": "transformer/diffusion_pytorch_model.fp16.safetensors", "bytes": 1183558264, "sha256": "4616f5f26161c4f072083edcd9b771b8a840057602e928dca00c8a05175b22a3"}, + {"path": "vae/diffusion_pytorch_model.fp16.safetensors", "bytes": 1249044836, "sha256": "15a4b09e56d95b768a0ec9da50b702e21d920333fc9b3480d66bb5c7fad9d87f"} + ], + "framework": "official Sana repository or Diffusers SanaPipeline; official ComfyUI guide requires ComfyUI_ExtraModels custom nodes and a custom VAE", + "apple_silicon": "not permitted by the NVIDIA License; no download, training, or inference on this host", + "lora": "official Diffusers Sana LoRA/DreamBooth training path exists, but cannot be used under the current host/license contract", + "custom_nodes": true, + "status": "fail_closed_license_and_platform" + } + ], + "baseline_reference": { + "model": "hcs.sd15-teaching-illustration-fp16", + "source": "Phase 2C.1 SD 1.5 ablation report in the prior ignored runtime worktree", + "not_rerun_in_this_loop": true + } +} diff --git a/benchmarks/phase2c2/generate_flat_cartoon_dataset.py b/benchmarks/phase2c2/generate_flat_cartoon_dataset.py new file mode 100644 index 0000000..eaf2a67 --- /dev/null +++ b/benchmarks/phase2c2/generate_flat_cartoon_dataset.py @@ -0,0 +1,229 @@ +"""Create the original, deterministic flat-cartoon LoRA training set. + +The only source is the drawing code below. SVG and PNG are written under the +ignored experiment directory; the manifest records the caption and generator +identity so the set can be regenerated without external assets. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +from PIL import Image, ImageDraw + + +WIDTH = HEIGHT = 512 +BACKGROUND = (248, 245, 236) +INK = (48, 58, 74) +PALETTE = { + "red": (224, 92, 85), + "blue": (76, 133, 190), + "yellow": (241, 190, 69), + "green": (93, 164, 117), + "purple": (141, 103, 178), + "orange": (232, 135, 65), + "cream": (255, 250, 231), +} + + +CASES: tuple[dict[str, Any], ...] = ( + {"id": "object_apple", "category": "single_object", "caption": "flat cartoon illustration of one red apple on a small table, centered object, plain warm background"}, + {"id": "object_book", "category": "single_object", "caption": "flat cartoon illustration of one blue book on a small table, centered object, plain warm background"}, + {"id": "person_wave", "category": "basic_action", "caption": "flat cartoon illustration of one child waving with one raised hand, standing alone, plain warm background"}, + {"id": "person_read", "category": "basic_action", "caption": "flat cartoon illustration of one child reading a blue book, seated at a table, plain warm background"}, + {"id": "space_left", "category": "spatial_relation", "caption": "flat cartoon illustration of a red apple to the left of a blue book on a table, clear horizontal separation"}, + {"id": "space_right", "category": "spatial_relation", "caption": "flat cartoon illustration of a blue cup to the right of a yellow ball on a table, clear horizontal separation"}, + {"id": "people_two_talk", "category": "daily_communication", "caption": "flat cartoon illustration of two children facing each other and talking beside a table, no text"}, + {"id": "people_two_greet", "category": "daily_communication", "caption": "flat cartoon illustration of two children greeting each other with raised hands, no text"}, + {"id": "people_three", "category": "person_count", "caption": "flat cartoon illustration of exactly three children standing in a row, all visible, plain background"}, + {"id": "class_pair", "category": "classroom_activity", "caption": "flat cartoon illustration of two children sharing one book at a classroom table, simple classroom objects"}, + {"id": "class_three", "category": "classroom_activity", "caption": "flat cartoon illustration of three children sitting around a classroom table, one teacher board without writing"}, + {"id": "object_cup", "category": "single_object", "caption": "flat cartoon illustration of one orange cup on a small table, centered object, plain warm background"}, +) + + +def _svg_wrap(body: str) -> str: + return ( + '' + f'{body}\n' + ) + + +def _svg_person(x: int, y: int, shirt: str, *, hand: str = "down", seated: bool = False) -> str: + px = x - 18 + py = y - 90 + body = f'' + body += f'' + if seated: + body += f'' + else: + body += f'' + if hand == "up": + body += f'' + body += f'' + elif hand == "left": + body += f'' + else: + body += f'' + return body + + +def _svg_object(kind: str, x: int, y: int) -> str: + if kind == "apple": + return f'' + if kind == "book": + return f'' + if kind == "cup": + return f'' + if kind == "ball": + return f'' + raise ValueError(kind) + + +def _draw_scene(case: dict[str, Any]) -> tuple[Image.Image, str]: + image = Image.new("RGB", (WIDTH, HEIGHT), BACKGROUND) + draw = ImageDraw.Draw(image) + # A consistent ground/table gives the set one restrained visual language. + draw.rounded_rectangle((76, 350, 436, 380), radius=10, fill=(213, 190, 157), outline=INK, width=4) + draw.line((116, 380, 100, 460), fill=INK, width=8) + draw.line((396, 380, 412, 460), fill=INK, width=8) + body = '' + body += '' + case_id = case["id"] + if case_id == "object_apple": + draw.ellipse((230, 275, 280, 325), fill=PALETTE["red"], outline=INK, width=4) + draw.arc((250, 245, 290, 285), 190, 300, fill=PALETTE["green"], width=7) + body += _svg_object("apple", 255, 300) + elif case_id == "object_book": + draw.polygon([(185, 280), (255, 265), (327, 280), (327, 335), (255, 320), (185, 335)], fill=PALETTE["blue"], outline=INK) + body += _svg_object("book", 255, 300) + elif case_id == "object_cup": + draw.polygon([(230, 275), (280, 275), (273, 330), (237, 330)], fill=PALETTE["orange"], outline=INK) + draw.arc((270, 285, 310, 320), 270, 90, fill=INK, width=6) + body += _svg_object("cup", 255, 300) + elif case_id == "person_wave": + draw.ellipse((236, 164, 276, 204), fill=(242, 199, 165), outline=INK, width=4) + draw.rounded_rectangle((218, 208, 294, 290), radius=14, fill=PALETTE["blue"], outline=INK, width=4) + draw.line((250, 290, 240, 350), fill=INK, width=8) + draw.line((270, 290, 280, 350), fill=INK, width=8) + draw.line((294, 230, 345, 180), fill=(242, 199, 165), width=10) + draw.ellipse((338, 168, 354, 184), fill=(242, 199, 165), outline=INK, width=3) + body += _svg_person(256, 204, "#4c85be", hand="up") + elif case_id == "person_read": + draw.ellipse((236, 164, 276, 204), fill=(242, 199, 165), outline=INK, width=4) + draw.rounded_rectangle((218, 208, 294, 290), radius=14, fill=PALETTE["green"], outline=INK, width=4) + draw.line((250, 290, 240, 350), fill=INK, width=8) + draw.line((270, 290, 280, 350), fill=INK, width=8) + draw.polygon([(294, 255), (347, 240), (347, 286), (294, 274)], fill=PALETTE["blue"], outline=INK) + body += _svg_person(256, 204, "#5da475", hand="left") + _svg_object("book", 320, 264) + elif case_id == "space_left": + draw.ellipse((160, 275, 210, 325), fill=PALETTE["red"], outline=INK, width=4) + draw.polygon([(300, 280), (350, 268), (400, 280), (400, 330), (350, 320), (300, 330)], fill=PALETTE["blue"], outline=INK) + body += _svg_object("apple", 185, 300) + _svg_object("book", 350, 300) + elif case_id == "space_right": + draw.ellipse((160, 275, 216, 331), fill=PALETTE["yellow"], outline=INK, width=4) + draw.polygon([(302, 275), (352, 275), (345, 330), (309, 330)], fill=PALETTE["orange"], outline=INK) + body += _svg_object("ball", 188, 303) + _svg_object("cup", 327, 303) + elif case_id in {"people_two_talk", "people_two_greet"}: + greet = case_id == "people_two_greet" + draw.ellipse((132, 170, 172, 210), fill=(242, 199, 165), outline=INK, width=4) + draw.rounded_rectangle((114, 214, 190, 296), radius=14, fill=PALETTE["purple"], outline=INK, width=4) + draw.ellipse((340, 170, 380, 210), fill=(226, 181, 145), outline=INK, width=4) + draw.rounded_rectangle((322, 214, 398, 296), radius=14, fill=PALETTE["green"], outline=INK, width=4) + for x in (145, 360): + draw.line((x, 296, x - 8, 350), fill=INK, width=8) + draw.line((x + 18, 296, x + 26, 350), fill=INK, width=8) + if greet: + draw.line((190, 240, 220, 200), fill=(242, 199, 165), width=10) + draw.line((322, 240, 292, 200), fill=(226, 181, 145), width=10) + else: + draw.line((190, 250, 230, 270), fill=(242, 199, 165), width=10) + draw.line((322, 250, 282, 270), fill=(226, 181, 145), width=10) + body += _svg_person(152, 210, "#8d67b2", hand="up" if greet else "left") + body += _svg_person(360, 210, "#5da475", hand="up" if greet else "left") + elif case_id == "people_three": + for x, color in ((150, "#e05c55"), (256, "#4c85be"), (362, "#f1be45")): + draw.ellipse((x - 20, 150, x + 20, 190), fill=(242, 199, 165), outline=INK, width=4) + draw.rounded_rectangle((x - 38, 194, x + 38, 276), radius=14, fill=color, outline=INK, width=4) + draw.line((x - 10, 276, x - 18, 340), fill=INK, width=8) + draw.line((x + 10, 276, x + 18, 340), fill=INK, width=8) + body += _svg_person(x, 190, color) + elif case_id == "class_pair": + draw.ellipse((144, 170, 184, 210), fill=(242, 199, 165), outline=INK, width=4) + draw.ellipse((328, 170, 368, 210), fill=(226, 181, 145), outline=INK, width=4) + draw.rounded_rectangle((126, 214, 202, 290), radius=14, fill=PALETTE["blue"], outline=INK, width=4) + draw.rounded_rectangle((310, 214, 386, 290), radius=14, fill=PALETTE["purple"], outline=INK, width=4) + draw.polygon([(218, 250), (294, 250), (294, 292), (218, 292)], fill=PALETTE["blue"], outline=INK) + body += _svg_person(164, 210, "#4c85be", hand="left", seated=True) + body += _svg_person(348, 210, "#8d67b2", hand="left", seated=True) + body += _svg_object("book", 256, 270) + elif case_id == "class_three": + draw.rectangle((170, 120, 342, 190), fill=PALETTE["cream"], outline=INK, width=4) + for x, color in ((170, "#e05c55"), (256, "#4c85be"), (342, "#5da475")): + draw.ellipse((x - 18, 180, x + 18, 216), fill=(242, 199, 165), outline=INK, width=4) + draw.rounded_rectangle((x - 32, 220, x + 32, 286), radius=12, fill=color, outline=INK, width=4) + body += _svg_person(x, 216, color, seated=True) + else: + raise ValueError(case_id) + return image, _svg_wrap(body) + + +def create_dataset(output_dir: Path) -> dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + records: list[dict[str, Any]] = [] + for case in CASES: + image, svg = _draw_scene(case) + png_path = output_dir / f"{case['id']}.png" + svg_path = output_dir / f"{case['id']}.svg" + image.save(png_path, format="PNG", optimize=True) + svg_path.write_text(svg, encoding="utf-8") + records.append( + { + **case, + "png": png_path.name, + "svg": svg_path.name, + "png_sha256": hashlib.sha256(png_path.read_bytes()).hexdigest(), + "svg_sha256": hashlib.sha256(svg_path.read_bytes()).hexdigest(), + "source": "original_programmatic_svg_and_pillow_renderer", + "rights": "project-authored; no external images, characters, or artist imitation", + } + ) + manifest = { + "schema": "hanclassstudio.phase2c2_flat_cartoon_dataset.v1", + "version": "1.0.0", + "created_at": "deterministic_generation", + "generator": "benchmarks/phase2c2/generate_flat_cartoon_dataset.py", + "generator_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "dimensions": [WIDTH, HEIGHT], + "count": len(records), + "records": records, + "license": "original project-authored procedural artwork", + } + (output_dir / "metadata.jsonl").write_text( + "".join( + json.dumps({"file_name": item["png"], "text": f"flatcartoonstyle, {item['caption']}"}, ensure_ascii=False) + "\n" + for item in records + ), + encoding="utf-8", + ) + (output_dir / "dataset-manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + manifest = create_dataset(args.output_dir) + print(json.dumps({"count": manifest["count"], "output_dir": str(args.output_dir)}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/phase2c2/model-evaluation.v1.json b/benchmarks/phase2c2/model-evaluation.v1.json new file mode 100644 index 0000000..9f2d38e --- /dev/null +++ b/benchmarks/phase2c2/model-evaluation.v1.json @@ -0,0 +1,136 @@ +{ + "schema": "hanclassstudio.phase2c2_model_evaluation.v1", + "version": "1.0.0", + "experiment_id": "phase2c2-lightweight-cartoon-model-evaluation", + "purpose": "Evaluate SSD-1B and Sana 0.6B with an original flat-cartoon LoRA without changing production defaults.", + "dimensions": {"width": 512, "height": 512}, + "runtime_contract": { + "production_runtime_id": "comfyui", + "production_runtime_version": "0.28.0", + "production_workflow_id": "hcs.teaching-illustration-sd15-core", + "evaluation_backend": "huggingface-diffusers", + "evaluation_backend_revision": "diffusers-v0.36.0", + "custom_nodes": false, + "arbitrary_graph": false + }, + "candidates": [ + { + "id": "ssd-1b", + "run_enabled": true, + "repository": "segmind/SSD-1B", + "revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", + "local_model_dir": "runtime/phase2c2-model-evaluation/models/ssd-1b-60987f37", + "license": "Apache-2.0", + "role": "formal_product_candidate" + }, + { + "id": "sana-600m-512", + "run_enabled": false, + "repository": "Efficient-Large-Model/Sana_600M_512px_diffusers", + "revision": "2defc07f5fb66d0c53ace051585e9a2cb83f8c15", + "local_model_dir": "runtime/phase2c2-model-evaluation/models/sana-600m-512-2defc07f", + "license": "NSCL v2-custom / NVIDIA License", + "role": "research_candidate", + "blocked_reason": "License limits use to non-commercial research/evaluation with NVIDIA Processors; this host is Apple Silicon MPS. Official ComfyUI path also requires custom nodes, which are outside this experiment." + } + ], + "variants": [ + { + "id": "base", + "name": "base model", + "prompt_profile": { + "id": "evaluation-plain-educational-v1", + "positive_prefix": "clear educational illustration, simple natural shapes, balanced composition, plain warm background, no written text", + "negative": "text, letters, words, captions, subtitles, watermark, logo, extra limbs, malformed hands, duplicate people, merged bodies, cropped subjects, distorted furniture, blurry, low resolution, oversaturated" + }, + "steps": 8, + "guidance_scale": 7.5, + "sampler": "euler", + "scheduler": "default_pipeline_scheduler", + "lora": false + }, + { + "id": "flat-cartoon-lora", + "name": "base plus original flat-cartoon LoRA", + "prompt_profile": { + "id": "flat-cartoon-evaluation-v1", + "positive_prefix": "flatcartoonstyle, clean flat two-dimensional educational cartoon, clear visual hierarchy, simple natural shapes, balanced composition, soft controlled colors, clear separation between people and objects, plain warm background, no written text", + "negative": "text, letters, words, captions, subtitles, watermark, logo, extra limbs, malformed hands, duplicate people, merged bodies, cropped subjects, distorted furniture, blurry, low resolution, oversaturated" + }, + "steps": 8, + "guidance_scale": 7.5, + "sampler": "euler", + "scheduler": "default_pipeline_scheduler", + "lora": true, + "lora_rank": 4, + "lora_alpha": 4 + } + ], + "cases": [ + { + "id": "single_object_apple", + "category": "single_object", + "prompt": "one red apple on a small table, centered and fully visible", + "expected": "one apple, one table, no people", + "environment": "plain classroom-like warm background", + "must_satisfy": ["exactly one apple", "apple fully visible", "no written text"], + "severe_failure": ["missing apple", "multiple apples", "text artifact"], + "seeds": [260101, 260201] + }, + { + "id": "spatial_between_books", + "category": "spatial_relation", + "prompt": "a green plant is between two blue books on a table, all three objects separated and fully visible", + "expected": "one plant between two books", + "environment": "simple indoor classroom table", + "must_satisfy": ["two books", "plant centered between books", "objects separated"], + "severe_failure": ["wrong spatial relation", "missing object", "merged objects"], + "seeds": [260112, 260212] + }, + { + "id": "two_person_greeting", + "category": "daily_communication", + "prompt": "exactly two children face each other and greet with raised hands, no text", + "expected": "two children greeting", + "environment": "plain indoor classroom background", + "must_satisfy": ["exactly two people", "facing interaction", "raised greeting hands"], + "severe_failure": ["wrong count", "wrong action", "text artifact"], + "seeds": [260116, 260216] + }, + { + "id": "one_person_waving", + "category": "person_action", + "prompt": "exactly one child waves with one raised hand while standing, fully visible", + "expected": "one child waving", + "environment": "plain warm classroom background", + "must_satisfy": ["exactly one person", "one raised waving hand", "full body visible"], + "severe_failure": ["wrong count", "wrong action", "anatomy defect"], + "seeds": [260104, 260204] + }, + { + "id": "exactly_three_people", + "category": "person_count", + "prompt": "exactly three children stand in one row, all three fully visible and separated", + "expected": "exactly three children", + "environment": "plain warm classroom background", + "must_satisfy": ["exactly three people", "all visible", "no merged bodies"], + "severe_failure": ["wrong count", "merged people", "cropped subject"], + "seeds": [260109, 260209] + }, + { + "id": "classroom_activity", + "category": "classroom_activity", + "prompt": "three children sit around a classroom table and share one open book, no writing on the board", + "expected": "three children, table, one open book", + "environment": "simple classroom with an empty board", + "must_satisfy": ["three children", "shared book", "classroom activity", "no board text"], + "severe_failure": ["wrong count", "missing book", "text artifact", "visually confusing"], + "seeds": [260115, 260215] + } + ], + "review_contract": { + "fields": ["visual_quality", "prompt_adherence", "object_count_accuracy", "action_accuracy", "spatial_relation_accuracy", "teaching_usability", "anatomy_quality", "needs_regeneration", "failure_tags", "reviewer_notes"], + "failure_tags": ["wrong_count", "wrong_action", "wrong_scene", "wrong_spatial_relation", "missing_object", "anatomy_defect", "text_artifact", "merged_people", "visually_confusing", "not_teaching_usable"], + "automatic_teacher_scores": false + } +} diff --git a/benchmarks/phase2c2/run_model_evaluation.py b/benchmarks/phase2c2/run_model_evaluation.py new file mode 100644 index 0000000..51ce61d --- /dev/null +++ b/benchmarks/phase2c2/run_model_evaluation.py @@ -0,0 +1,194 @@ +"""Real SSD-1B Diffusers pilot runner. + +Sana is intentionally rejected before this executor can load it: the official +Sana checkpoint license is NVIDIA-processor-only, while this host is Apple +Silicon. The runner is therefore an opt-in SSD-1B executor plus a shared +resumable contract used to preserve failed/paused state. +""" + +from __future__ import annotations + +import argparse +import io +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +import psutil +import torch +from PIL import Image + +from hcs_api.phase2c2_model_evaluation import ( + EvaluationRunner, + EvaluationTask, + aggregate_report, + load_spec, + write_review_package, +) + + +def _swap_usage() -> str: + try: + return subprocess.check_output(["sysctl", "-n", "vm.swapusage"], text=True, timeout=2).strip() + except (OSError, subprocess.SubprocessError): + return "unavailable" + + +def _mps_memory() -> dict[str, int | None]: + if not hasattr(torch, "mps") or not torch.backends.mps.is_available(): + return {"allocated_bytes": None, "driver_allocated_bytes": None} + try: + return { + "allocated_bytes": int(torch.mps.current_allocated_memory()), + "driver_allocated_bytes": int(torch.mps.driver_allocated_memory()), + } + except RuntimeError: + return {"allocated_bytes": None, "driver_allocated_bytes": None} + + +def _model_identity(model_dir: Path) -> dict[str, Any]: + files: dict[str, dict[str, Any]] = {} + for path in sorted(model_dir.rglob("*.safetensors")): + if not path.is_file(): + continue + stat = path.stat() + files[path.relative_to(model_dir).as_posix()] = {"size_bytes": stat.st_size} + return {"model_dir": str(model_dir), "safetensors": files} + + +class SSDExecutor: + def __init__(self, model_dir: Path, lora_dir: Path | None, *, width: int, height: int) -> None: + from diffusers import StableDiffusionXLPipeline + + self.width = width + self.height = height + self.lora_dir = lora_dir + self._lora_loaded = False + self.process = psutil.Process() + load_started = time.monotonic() + self.pipeline = StableDiffusionXLPipeline.from_pretrained( + model_dir, + torch_dtype=torch.float16, + variant="fp16", + use_safetensors=True, + local_files_only=True, + ) + self.pipeline.to("mps") + self.pipeline.enable_attention_slicing() + self.pipeline.set_progress_bar_config(disable=True) + self.load_seconds = time.monotonic() - load_started + self.load_rss = self.process.memory_info().rss + self.load_mps = _mps_memory() + + def _ensure_variant(self, task: EvaluationTask) -> None: + wants_lora = task.variant_id == "flat-cartoon-lora" + if not wants_lora or self._lora_loaded: + return + if self.lora_dir is None or not self.lora_dir.is_dir(): + raise RuntimeError("flat-cartoon LoRA directory is unavailable") + self.pipeline.load_lora_weights(self.lora_dir, weight_name="pytorch_lora_weights.safetensors") + self._lora_loaded = True + + def __call__(self, task: EvaluationTask) -> dict[str, Any]: + self._ensure_variant(task) + before_rss = self.process.memory_info().rss + before_mps = _mps_memory() + started = time.monotonic() + generator = torch.Generator(device="cpu").manual_seed(task.seed) + with torch.inference_mode(): + result = self.pipeline( + prompt=task.plan["positive_prompt"], + negative_prompt=task.plan["negative_prompt"], + width=self.width, + height=self.height, + num_inference_steps=task.plan["steps"], + guidance_scale=task.plan["guidance_scale"], + generator=generator, + ) + image = result.images[0] + buffer = io.BytesIO() + image.save(buffer, format="PNG", optimize=True) + payload = buffer.getvalue() + extrema = image.convert("RGB").getextrema() + near_solid = all((high - low) <= 4 for low, high in extrema) + ended = time.monotonic() + return { + "png_bytes": payload, + "technical": { + "duration_seconds": round(ended - started, 3), + "load_seconds": round(self.load_seconds, 3), + "rss_before_bytes": before_rss, + "rss_after_bytes": self.process.memory_info().rss, + "rss_load_bytes": self.load_rss, + "rss_delta_bytes": self.process.memory_info().rss - before_rss, + "mps_before": before_mps, + "mps_after": _mps_memory(), + "mps_load": self.load_mps, + "swap_usage": _swap_usage(), + "near_solid_warning": near_solid, + "device": "mps", + }, + } + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c2/model-evaluation.v1.json")) + parser.add_argument("--model-dir", type=Path, default=Path("runtime/phase2c2-model-evaluation/models/ssd-1b-60987f37")) + parser.add_argument("--lora-dir", type=Path) + parser.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c2-model-evaluation/ssd-pilot")) + parser.add_argument("--state-path", type=Path) + parser.add_argument("--stop-after", type=int) + parser.add_argument("--max-attempts", type=int, default=1) + parser.add_argument("--review-package", action="store_true") + parser.add_argument("--report-only", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + spec = load_spec(args.spec) + args.output_dir.mkdir(parents=True, exist_ok=True) + state_path = args.state_path or args.output_dir / "run-state.json" + identity = { + "backend": "huggingface-diffusers", + "diffusers_revision": "v0.36.0", + "torch_version": torch.__version__, + "device": "mps", + "runtime_host": "macos-arm64-16gb", + "candidate_id": "ssd-1b", + "model_revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", + "model_files": _model_identity(args.model_dir), + "lora_dir": str(args.lora_dir) if args.lora_dir else None, + } + runner = EvaluationRunner(spec, args.output_dir, state_path, identity) + if not args.report_only: + if not args.model_dir.is_dir(): + raise SystemExit(f"SSD model directory missing: {args.model_dir}") + executor = SSDExecutor( + args.model_dir, + args.lora_dir, + width=spec["dimensions"]["width"], + height=spec["dimensions"]["height"], + ) + state = runner.run(executor, max_attempts=max(1, args.max_attempts), stop_after=args.stop_after) + else: + state = runner.state + if args.review_package: + write_review_package(spec, state, args.output_dir) + report = aggregate_report(spec, state, args.output_dir, blockers=[ + { + "candidate_id": "sana-600m-512", + "status": "fail_closed", + "reason": "NVIDIA License restricts the checkpoint to NVIDIA Processors; current host is Apple Silicon.", + } + ]) + print(json.dumps({"status": state["status"], "succeeded": report["succeeded"], "failed": report["failed"], "pending": report["pending"]}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/phase2c2/train_flat_cartoon_lora.py b/benchmarks/phase2c2/train_flat_cartoon_lora.py new file mode 100644 index 0000000..c74afb3 --- /dev/null +++ b/benchmarks/phase2c2/train_flat_cartoon_lora.py @@ -0,0 +1,76 @@ +"""Run the pinned official Diffusers SDXL LoRA trainer on the original set.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import time +from pathlib import Path + + +def build_command(args: argparse.Namespace) -> list[str]: + script = args.diffusers_source / "examples/text_to_image/train_text_to_image_lora_sdxl.py" + if not script.is_file(): + raise SystemExit(f"official Diffusers trainer is missing: {script}") + return [ + str(args.python), "-m", "accelerate.commands.launch", + "--num_processes", "1", + str(script), + "--pretrained_model_name_or_path", str(args.model_dir.resolve()), + "--train_data_dir", str(args.dataset_dir.resolve()), + "--caption_column", "text", + "--resolution", "512", + "--center_crop", + "--random_flip", + "--train_batch_size", "1", + "--gradient_accumulation_steps", "2", + "--gradient_checkpointing", + "--max_train_steps", str(args.max_train_steps), + "--checkpointing_steps", str(max(1, args.max_train_steps // 2)), + "--checkpoints_total_limit", "2", + "--learning_rate", "1e-4", + "--lr_scheduler", "constant", + "--lr_warmup_steps", "0", + "--mixed_precision", "fp16", + "--rank", "4", + "--seed", "4242", + "--output_dir", str(args.output_dir.resolve()), + "--report_to", "none", + "--dataloader_num_workers", "0", + ] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--python", type=Path, default=Path("runtime/phase2c2-model-evaluation/env/bin/python")) + parser.add_argument("--diffusers-source", type=Path, default=Path("runtime/phase2c2-model-evaluation/diffusers-source")) + parser.add_argument("--model-dir", type=Path, default=Path("runtime/phase2c2-model-evaluation/models/ssd-1b-60987f37")) + parser.add_argument("--dataset-dir", type=Path, default=Path("runtime/phase2c2-model-evaluation/dataset")) + parser.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c2-model-evaluation/ssd-flat-cartoon-lora")) + parser.add_argument("--max-train-steps", type=int, default=12) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + command = build_command(args) + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "command.json").write_text(json.dumps({"command": command, "created_at": time.time()}, indent=2) + "\n", encoding="utf-8") + if args.dry_run: + print(json.dumps({"command": command}, ensure_ascii=False)) + return + started = time.monotonic() + log_path = args.output_dir / "training.log" + env = {**os.environ, "PYTORCH_ENABLE_MPS_FALLBACK": "1"} + with log_path.open("w", encoding="utf-8") as log: + completed = subprocess.run(command, cwd=args.diffusers_source, env=env, stdout=log, stderr=subprocess.STDOUT, check=False) + duration = round(time.monotonic() - started, 3) + lora_files = [{"path": str(path.relative_to(args.output_dir)), "size_bytes": path.stat().st_size} for path in args.output_dir.rglob("*.safetensors") if path.is_file()] + metrics = {"status": "succeeded" if completed.returncode == 0 else "failed", "returncode": completed.returncode, "duration_seconds": duration, "lora_files": lora_files, "command": command, "official_diffusers_source": str(args.diffusers_source)} + (args.output_dir / "training-metrics.json").write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + print(json.dumps(metrics, ensure_ascii=False)) + if completed.returncode: + raise SystemExit(completed.returncode) + + +if __name__ == "__main__": + main() From edea54302c17c3de5bffd28ef97ddeab035697b5 Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:06:54 +0700 Subject: [PATCH 2/3] fix(phase2c2): reject blank outputs and upcast MPS VAE --- apps/api/src/hcs_api/phase2c2_model_evaluation.py | 11 +++++------ apps/api/tests/test_phase2c2_model_evaluation.py | 1 - benchmarks/phase2c2/generate_flat_cartoon_dataset.py | 1 - benchmarks/phase2c2/run_model_evaluation.py | 8 ++++---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/apps/api/src/hcs_api/phase2c2_model_evaluation.py b/apps/api/src/hcs_api/phase2c2_model_evaluation.py index ae2384f..b7427e3 100644 --- a/apps/api/src/hcs_api/phase2c2_model_evaluation.py +++ b/apps/api/src/hcs_api/phase2c2_model_evaluation.py @@ -14,14 +14,12 @@ import os import shutil import struct -import time -import uuid import zlib +from collections.abc import Callable, Iterable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Any, Callable, Iterable - +from typing import Any SCHEMA = "hanclassstudio.phase2c2_model_evaluation.v1" REVIEW_SCHEMA = "hanclassstudio.phase2c2_model_evaluation_review.v1" @@ -123,7 +121,6 @@ def _request_for(spec: dict[str, Any], candidate_id: str, variant_id: str, case: candidate = _candidate(spec, candidate_id) variant = _variant(spec, variant_id) task_id = f"{candidate_id}:{variant_id}:{case['id']}:{seed}" - task_hash = sha256_bytes(task_id.encode("utf-8")) asset_id = f"eval-{candidate_id}-{variant_id}-{case['id']}-{seed}"[:78] prompt_profile = variant["prompt_profile"] request = { @@ -323,6 +320,8 @@ def _persist(self, task: EvaluationTask, rendered: dict[str, Any]) -> dict[str, if not isinstance(png, bytes): raise ModelEvaluationError("executor did not return PNG bytes") tech = verify_png(png, expected_width=self.spec["dimensions"]["width"], expected_height=self.spec["dimensions"]["height"]) + if (rendered.get("technical") or {}).get("near_solid_warning"): + raise ModelEvaluationError("technical precheck rejected a near-solid generated image") task_dir = self.output_dir / "images" / _safe_name(task.candidate_id) / _safe_name(task.variant_id) provenance_dir = self.output_dir / "provenance" image_name = f"{_safe_name(task.case_id)}-{task.seed}.png" @@ -435,7 +434,7 @@ def run(self, executor: Callable[[EvaluationTask], dict[str, Any]], *, max_attem record["failure"] = None self._save() break - except Exception as exc: # individual failure never aborts the batch + except Exception as exc: # noqa: BLE001 - isolate each task failure and continue the batch record["failure"] = {"type": type(exc).__name__, "message": str(exc), "at": utc_now()} record["status"] = "failed" self._save() diff --git a/apps/api/tests/test_phase2c2_model_evaluation.py b/apps/api/tests/test_phase2c2_model_evaluation.py index 2704bfa..b072a1d 100644 --- a/apps/api/tests/test_phase2c2_model_evaluation.py +++ b/apps/api/tests/test_phase2c2_model_evaluation.py @@ -14,7 +14,6 @@ write_review_package, ) - ROOT = Path(__file__).parents[3] SPEC = ROOT / "benchmarks/phase2c2/model-evaluation.v1.json" diff --git a/benchmarks/phase2c2/generate_flat_cartoon_dataset.py b/benchmarks/phase2c2/generate_flat_cartoon_dataset.py index eaf2a67..7ce14f1 100644 --- a/benchmarks/phase2c2/generate_flat_cartoon_dataset.py +++ b/benchmarks/phase2c2/generate_flat_cartoon_dataset.py @@ -15,7 +15,6 @@ from PIL import Image, ImageDraw - WIDTH = HEIGHT = 512 BACKGROUND = (248, 245, 236) INK = (48, 58, 74) diff --git a/benchmarks/phase2c2/run_model_evaluation.py b/benchmarks/phase2c2/run_model_evaluation.py index 51ce61d..2691991 100644 --- a/benchmarks/phase2c2/run_model_evaluation.py +++ b/benchmarks/phase2c2/run_model_evaluation.py @@ -11,17 +11,13 @@ import argparse import io import json -import os import subprocess -import sys import time from pathlib import Path from typing import Any import psutil import torch -from PIL import Image - from hcs_api.phase2c2_model_evaluation import ( EvaluationRunner, EvaluationTask, @@ -78,6 +74,9 @@ def __init__(self, model_dir: Path, lora_dir: Path | None, *, width: int, height local_files_only=True, ) self.pipeline.to("mps") + # MPS fp16 VAE decode can overflow to NaN/black for SDXL-family + # checkpoints; keep the denoiser fp16 but decode in float32. + self.pipeline.vae.to(dtype=torch.float32) self.pipeline.enable_attention_slicing() self.pipeline.set_progress_bar_config(disable=True) self.load_seconds = time.monotonic() - load_started @@ -164,6 +163,7 @@ def main() -> None: "model_revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", "model_files": _model_identity(args.model_dir), "lora_dir": str(args.lora_dir) if args.lora_dir else None, + "vae_dtype": "float32", } runner = EvaluationRunner(spec, args.output_dir, state_path, identity) if not args.report_only: From efbd7907a6d13199cb3a99c8f44317790f3f1b7f Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:26:49 +0700 Subject: [PATCH 3/3] docs(phase2c2): record SSD and Sana model evaluation --- .../src/hcs_api/phase2c2_model_evaluation.py | 19 +- .../tests/test_phase2c2_model_evaluation.py | 3 + .../phase2c2/train_flat_cartoon_lora.py | 29 ++- docs/phase2c2-model-evaluation.md | 184 ++++++++++++++++++ 4 files changed, 230 insertions(+), 5 deletions(-) create mode 100644 docs/phase2c2-model-evaluation.md diff --git a/apps/api/src/hcs_api/phase2c2_model_evaluation.py b/apps/api/src/hcs_api/phase2c2_model_evaluation.py index b7427e3..187929b 100644 --- a/apps/api/src/hcs_api/phase2c2_model_evaluation.py +++ b/apps/api/src/hcs_api/phase2c2_model_evaluation.py @@ -539,7 +539,22 @@ def aggregate_report(spec: dict[str, Any], state: dict[str, Any], output_dir: Pa report["candidate_results"][cid] = {"total": len(subset), "succeeded": sum(item["status"] == "succeeded" for item in subset), "failed": sum(item["status"] == "failed" for item in subset), "variants": {}} for variant in spec["variants"]: v = [item for item in subset if item["variant_id"] == variant["id"]] - durations = [item["result"]["technical"].get("duration_seconds") for item in v if item.get("result") and item["result"]["technical"].get("duration_seconds") is not None] - report["candidate_results"][cid]["variants"][variant["id"]] = {"total": len(v), "succeeded": sum(item["status"] == "succeeded" for item in v), "failed": sum(item["status"] == "failed" for item in v), "mean_duration_seconds": (sum(durations) / len(durations) if durations else None)} + successful = [item for item in v if item.get("status") == "succeeded" and item.get("result")] + technical = [item["result"].get("technical", {}) for item in successful] + durations = [item.get("duration_seconds") for item in technical if item.get("duration_seconds") is not None] + driver_allocations = [item.get("mps_after", {}).get("driver_allocated_bytes") for item in technical if item.get("mps_after", {}).get("driver_allocated_bytes") is not None] + rss_after = [item.get("rss_after_bytes") for item in technical if item.get("rss_after_bytes") is not None] + report["candidate_results"][cid]["variants"][variant["id"]] = { + "total": len(v), + "succeeded": sum(item["status"] == "succeeded" for item in v), + "failed": sum(item["status"] == "failed" for item in v), + "mean_duration_seconds": (sum(durations) / len(durations) if durations else None), + "duration_min_seconds": min(durations) if durations else None, + "duration_max_seconds": max(durations) if durations else None, + "max_mps_driver_allocated_bytes": max(driver_allocations) if driver_allocations else None, + "max_rss_after_bytes": max(rss_after) if rss_after else None, + "near_solid_warnings": sum(bool(item.get("near_solid_warning")) for item in technical), + "technical_success_rate": (len(successful) / len(v) if v else None), + } write_json(output_dir / "model-evaluation-report.json", report) return report diff --git a/apps/api/tests/test_phase2c2_model_evaluation.py b/apps/api/tests/test_phase2c2_model_evaluation.py index b072a1d..09ac1c8 100644 --- a/apps/api/tests/test_phase2c2_model_evaluation.py +++ b/apps/api/tests/test_phase2c2_model_evaluation.py @@ -94,4 +94,7 @@ def execute(task): assert state["status"] == "completed" report = aggregate_report(spec, state, output) assert report["failed"] == 0 + variant_report = report["candidate_results"]["ssd-1b"]["variants"]["base"] + assert variant_report["technical_success_rate"] == 1.0 + assert variant_report["mean_duration_seconds"] == 0.1 assert sha256_json(identity) == state["identity_sha256"] diff --git a/benchmarks/phase2c2/train_flat_cartoon_lora.py b/benchmarks/phase2c2/train_flat_cartoon_lora.py index c74afb3..b687041 100644 --- a/benchmarks/phase2c2/train_flat_cartoon_lora.py +++ b/benchmarks/phase2c2/train_flat_cartoon_lora.py @@ -11,12 +11,14 @@ def build_command(args: argparse.Namespace) -> list[str]: - script = args.diffusers_source / "examples/text_to_image/train_text_to_image_lora_sdxl.py" + script = args.diffusers_source.resolve() / "examples/text_to_image/train_text_to_image_lora_sdxl.py" + python_path = args.python if args.python.is_absolute() else (Path.cwd() / args.python).absolute() if not script.is_file(): raise SystemExit(f"official Diffusers trainer is missing: {script}") return [ - str(args.python), "-m", "accelerate.commands.launch", + str(python_path), "-m", "accelerate.commands.launch", "--num_processes", "1", + "--mixed_precision", "fp16", str(script), "--pretrained_model_name_or_path", str(args.model_dir.resolve()), "--train_data_dir", str(args.dataset_dir.resolve()), @@ -37,11 +39,31 @@ def build_command(args: argparse.Namespace) -> list[str]: "--rank", "4", "--seed", "4242", "--output_dir", str(args.output_dir.resolve()), - "--report_to", "none", + "--report_to", "tensorboard", "--dataloader_num_workers", "0", ] +def ensure_training_aliases(model_dir: Path) -> None: + """The official SDXL trainer asks Transformers for model.safetensors. + + The pinned snapshot intentionally keeps only the fp16 variant. Hardlinks + expose the same bytes under Transformers' expected names without doubling + disk use or changing the downloaded model content. + """ + aliases = ( + ("text_encoder", "model.fp16.safetensors", "model.safetensors"), + ("text_encoder_2", "model.fp16.safetensors", "model.safetensors"), + ("unet", "diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.safetensors"), + ("vae", "diffusion_pytorch_model.fp16.safetensors", "diffusion_pytorch_model.safetensors"), + ) + for component, source_name, target_name in aliases: + source = model_dir / component / source_name + target = model_dir / component / target_name + if source.is_file() and not target.exists(): + os.link(source, target) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--python", type=Path, default=Path("runtime/phase2c2-model-evaluation/env/bin/python")) @@ -52,6 +74,7 @@ def main() -> None: parser.add_argument("--max-train-steps", type=int, default=12) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() + ensure_training_aliases(args.model_dir) command = build_command(args) args.output_dir.mkdir(parents=True, exist_ok=True) (args.output_dir / "command.json").write_text(json.dumps({"command": command, "created_at": time.time()}, indent=2) + "\n", encoding="utf-8") diff --git a/docs/phase2c2-model-evaluation.md b/docs/phase2c2-model-evaluation.md new file mode 100644 index 0000000..41dd5c1 --- /dev/null +++ b/docs/phase2c2-model-evaluation.md @@ -0,0 +1,184 @@ +# Phase 2C.2 lightweight teaching-image model evaluation + +This document records the first controlled evaluation of two lightweight cartoon +image routes on the project’s Apple Silicon opt-in host. It is an evaluation +record, not a production-model change. The fixed production Model Package, +Workflow Pack, Provider Hub defaults, and teacher UI were not changed. + +## Decision summary + +The SSD-1B route completed a real base-model and original-LoRA pilot on the +16 GB M4 host. The LoRA produced a clear flat-cartoon style shift, but the +pilot still showed wrong counts, actions, and spatial relations. It is therefore +not promoted to the formal product candidate from this evidence alone. The +pilot stopped before the larger benchmark because it did not establish a clear +teaching-content improvement over the existing SD 1.5 reference. Teacher review +is still required; no teacher score or usability conclusion is inferred here. + +Sana 0.6B was fail-closed before download, training, or inference. Its official +checkpoint license restricts use to non-commercial research/evaluation with +NVIDIA Processors, while this experiment host is Apple Silicon. The official +ComfyUI route also requires `ComfyUI_ExtraModels` custom nodes and a custom VAE, +outside this experiment’s no-custom-node boundary. It remains a blocked +research lead, not a tested product result. + +## Fixed candidate audit + +The machine-readable audit is +[`benchmarks/phase2c2/candidate-audit.v1.json`](../benchmarks/phase2c2/candidate-audit.v1.json). +All downloaded SSD component hashes were recomputed locally after download. +Sana hashes are recorded from the pinned upstream snapshot; the files were not +downloaded on this host. + +| Route | Fixed upstream identity | License / role | Selected fp16 bytes | Result | +| --- | --- | --- | ---: | --- | +| SSD-1B | `segmind/SSD-1B` @ `60987f37e94cd59c36b1cba832b9f97b57395a10` | Apache-2.0 model card; formal candidate | 4,465,653,694 (~4.16 GiB) | downloaded, verified, trained, inferred | +| Sana 0.6B | `Efficient-Large-Model/Sana_600M_512px_diffusers` @ `2defc07f5fb66d0c53ace051585e9a2cb83f8c15` | NVIDIA License / NSCL v2-custom; research candidate | 7,699,969,740 (~7.17 GiB) | fail-closed on Apple Silicon | + +SSD selected component identities: + +| Component | Bytes | SHA-256 | +| --- | ---: | --- | +| `text_encoder/model.fp16.safetensors` | 246,144,864 | `5487ea0eee9c9a9bff8abd097908d4deff3ae1fa87b3b67397f8b9538139d447` | +| `text_encoder_2/model.fp16.safetensors` | 1,389,382,880 | `d3df577f6e3799c8e1bd9b40e30133710e02e8e25d0ce48cdcc790e7dfe12d6d` | +| `unet/diffusion_pytorch_model.fp16.safetensors` | 2,662,790,608 | `40d8ea9159f3e875278dacc7879442d58c45850cf13c62f5e26681061c51829a` | +| `vae/diffusion_pytorch_model.fp16.safetensors` | 167,335,342 | `6353737672c94b96174cb590f711eac6edf2fcce5b6e91aa9d73c5adc589ee48` | + +The original SSD model card is at +[`segmind/SSD-1B`](https://huggingface.co/segmind/SSD-1B). It describes the +checkpoint as an SDXL distilled model, documents Diffusers/LoRA usage, and +lists GRIT and a Midjourney-derived scrape among its training data. Apache-2.0 +does not remove the separate training-data provenance review risk. + +The Sana model card and license are +[`Sana_600M_512px_diffusers`](https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers) +and its pinned +[`LICENSE.txt`](https://huggingface.co/Efficient-Large-Model/Sana_600M_512px_diffusers/blob/2defc07f5fb66d0c53ace051585e9a2cb83f8c15/LICENSE.txt). +The official +[`Sana ComfyUI guide`](https://nvlabs.github.io/Sana/docs/ComfyUI/comfyui/) +requires the extra-model custom-node repository; that route was not executed. + +## Original training set and LoRA + +`benchmarks/phase2c2/generate_flat_cartoon_dataset.py` is the sole source of a +deterministic 12-scene set. It writes SVG plus PNG and a captioned +`metadata.jsonl` under the ignored runtime directory. The scenes cover three +single objects, two basic actions, two spatial relations, two-person +communication, exact three-person count, and two classroom activities. The +manifest records generator hash, per-file hashes, captions, and +`project-authored; no external images, characters, or artist imitation` rights. + +The real SSD training used the pinned official Diffusers source (v0.36.0), +`train_text_to_image_lora_sdxl.py`, 512px, 12 steps, rank 4, alpha 4, fp16, +batch 1, gradient accumulation 2, gradient checkpointing, learning rate +`1e-4`, seed `4242`, and no xformers/bitsandbytes. It completed in 96.382 s +without an OOM or runtime crash. The final LoRA is 10,910,072 bytes with +SHA-256 `03dd1968d37e7fdb5f1d86a5ad3ac09a961acd37b39353aed27f398e6266f785`. +Hardlink aliases expose the pinned fp16 files under the official trainer’s +expected names; they do not duplicate model bytes. + +This small set is sufficient to test the loop, not to establish a formal +product-quality style artifact. The LoRA should not be added to the production +Model Package without a rights review and teacher-quality review. + +## Pilot contract and execution + +The fixed benchmark contract is +[`benchmarks/phase2c2/model-evaluation.v1.json`](../benchmarks/phase2c2/model-evaluation.v1.json): +512×512, six teaching cases, two fixed seeds per case, and two variants (base +and base + `flat-cartoon-lora`). Cases cover a single object, a spatial +relation, two-person greeting, one-person action, exact three-person count, and +classroom activity. The evaluator binds each case to a +`TeachingImageRequest`, execution plan, artifact, provenance record, and Asset +Manifest entry. It is resumable, idempotent, retry-bounded, and invalidates +unfinished work when the model/runtime/LoRA identity changes. + +The real opt-in command was: + +```sh +PYTHONPATH=apps/api/src PYTORCH_ENABLE_MPS_FALLBACK=1 \ + runtime/phase2c2-model-evaluation/env/bin/python \ + benchmarks/phase2c2/run_model_evaluation.py \ + --lora-dir runtime/phase2c2-model-evaluation/ssd-flat-cartoon-lora \ + --max-attempts 1 --review-package +``` + +The first fp16-VAE attempt produced a near-solid black PNG. The evaluator +rejected that image, persisted an invalidated state, and the opt-in executor +was corrected to decode the VAE in float32 while retaining the denoiser/text +encoders in fp16. The pilot was then rerun under the new identity; stale black +output was not reused. + +## Real pilot results + +| Variant | Tasks | Technical success | Mean generation | Min–max | Max observed MPS driver allocation | +| --- | ---: | ---: | ---: | ---: | ---: | +| SSD-1B base | 12 | 12/12 | 11.480 s | 10.761–13.696 s | 8,250,933,248 bytes | +| SSD-1B + original LoRA | 12 | 12/12 | 15.220 s | 14.971–15.489 s | 8,284,487,680 bytes | + +All 24 PNGs passed signature, CRC, dimension, and SHA checks. All 24 +provenance records bind the current model identity, request SHA, execution-plan +SHA, artifact SHA, and Asset Manifest entry. No task failed, timed out, or +reported a near-solid output in the final run. The process recorded RSS and +`vm.swapusage` per image; the machine was already under memory pressure, so +those values are evidence of the observed host state, not a portable minimum +hardware claim. + +The ignored review package is +`runtime/phase2c2-model-evaluation/ssd-pilot/review-package/`. +Open `comparison.html` for the 24-image comparison. The package includes +`teacher-reviews.pending.json` and `teacher-reviews.csv`; all 24 records are +`pending_review`, with zero teacher reviews imported. + +### Provisional technical observations (not teacher conclusions) + +- Base SSD images were often coherent and more photographic/soft-illustration + than the target flat-cartoon style. +- The LoRA reliably shifted the palette and rendering toward a clean flat + cartoon look and improved visual consistency for simple scenes. +- Both variants still produced wrong counts or actions in people cases; the + one-person wave and exact-three-person cases are especially sensitive to + over/under-counting and hand/body errors. +- The plant-between-books case did not reliably express the required relation; + the LoRA changes style more consistently than it improves relation control. +- A classroom scene can look presentation-ready at a glance while still + missing the requested number of pupils or shared-book arrangement. + +These are engineering observations used to decide whether to expand the run; +they are not automatic teacher scores. The quality fields and failure tags +remain empty until a real teacher uses the review package. + +## SD 1.5 reference and decision + +The prior Phase 2C.1 SD 1.5 ablation is a technical reference only and was not +rerun in this loop. Its current-baseline configuration completed 18/18 images +at a mean 35.011 s, with teacher review still pending. The backends, prompts, +steps, and experimental purpose are not identical, so this is not an +apples-to-apples quality claim. SSD is materially faster in this pilot, but no +quality superiority can be declared without teacher scores. + +The current decision is: + +1. Do not replace the production SD 1.5 Model Package or Workflow Pack. +2. Do not promote this small LoRA to a fixed production artifact. It is a + promising style probe, not evidence of better content control. +3. Keep SSD-1B as an experimental candidate for teacher review and, if needed, + a larger rights-cleared dataset; prefer content-control improvements before + prompt-only or style-only tuning. +4. Keep Sana fail-closed. Revisit only on an authorized NVIDIA host with an + explicit license/product decision and a reviewed, no-custom-node execution + path. Do not “work around” the processor restriction on Apple hardware. +5. If the next review confirms the count/action/spatial failures, evaluate a + stronger model rather than expanding this LoRA blindly. A small additional + fixed workflow pack is a later, separate decision—not part of this PR. + +## Reproduction and boundaries + +All model, LoRA, dataset, images, caches, reports, virtualenv, and runtime +files are under `runtime/phase2c2-model-evaluation/` and ignored by Git. The +loop state is `.workbuddy/phase2c2-model-evaluation-state.md`, also ignored. +No production configuration, arbitrary ComfyUI graph, custom node, cloud +fallback, or teacher score was added. Sana has no generated image or training +attempt because fail-closed is the required result under the current license +and host boundary. +