From 4398305d6dda202b82d2c5dc8bf0a8619337ae39 Mon Sep 17 00:00:00 2001 From: Hsueh0216 <105940021+Hsueh0216@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:26:11 +0700 Subject: [PATCH] feat(phase2c3): add SSD prompt and parameter ablation --- benchmarks/phase2c3/ablation.v1.json | 210 +++++++ benchmarks/phase2c3/ablation_contract.py | 554 ++++++++++++++++++ benchmarks/phase2c3/run_ablation.py | 227 +++++++ benchmarks/phase2c3/selection.v1.json | 23 + benchmarks/phase2c3/test_ablation_contract.py | 69 +++ docs/phase2c3-prompt-parameter-ablation.md | 165 ++++++ 6 files changed, 1248 insertions(+) create mode 100644 benchmarks/phase2c3/ablation.v1.json create mode 100644 benchmarks/phase2c3/ablation_contract.py create mode 100644 benchmarks/phase2c3/run_ablation.py create mode 100644 benchmarks/phase2c3/selection.v1.json create mode 100644 benchmarks/phase2c3/test_ablation_contract.py create mode 100644 docs/phase2c3-prompt-parameter-ablation.md diff --git a/benchmarks/phase2c3/ablation.v1.json b/benchmarks/phase2c3/ablation.v1.json new file mode 100644 index 0000000..fe0446c --- /dev/null +++ b/benchmarks/phase2c3/ablation.v1.json @@ -0,0 +1,210 @@ +{ + "schema": "hanclassstudio.phase2c3_prompt_parameter_ablation.v1", + "version": "1.0.0", + "experiment_id": "phase2c3-ssd1b-prompt-parameter-ablation", + "purpose": "Controlled prompt and sampling-parameter ablation for the existing SSD-1B plus original flat-cartoon LoRA route.", + "host_contract": { + "operating_system": "macos", + "architecture": "arm64", + "memory_bytes": 17179869184, + "device": "mps" + }, + "runtime_contract": { + "backend": "huggingface-diffusers", + "diffusers_revision": "v0.36.0", + "model_revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", + "width": 512, + "height": 512, + "variant": "ssd-1b-plus-original-flat-cartoon-lora", + "production_defaults_changed": false, + "custom_nodes": false, + "arbitrary_graph": false + }, + "model": { + "id": "ssd-1b", + "repository": "segmind/SSD-1B", + "revision": "60987f37e94cd59c36b1cba832b9f97b57395a10", + "license": "Apache-2.0", + "selected_files": [ + {"path": "text_encoder/model.fp16.safetensors", "bytes": 246144864, "sha256": "5487ea0eee9c9a9bff8abd097908d4deff3ae1fa87b3b67397f8b9538139d447"}, + {"path": "text_encoder_2/model.fp16.safetensors", "bytes": 1389382880, "sha256": "d3df577f6e3799c8e1bd9b40e30133710e02e8e25d0ce48cdcc790e7dfe12d6d"}, + {"path": "unet/diffusion_pytorch_model.fp16.safetensors", "bytes": 2662790608, "sha256": "40d8ea9159f3e875278dacc7879442d58c45850cf13c62f5e26681061c51829a"}, + {"path": "vae/diffusion_pytorch_model.fp16.safetensors", "bytes": 167335342, "sha256": "6353737672c94b96174cb590f711eac6edf2fcce5b6e91aa9d73c5adc589ee48"} + ] + }, + "lora": { + "id": "flat-cartoon-lora", + "source": "project-authored procedural SVG/Pillow dataset; official Diffusers SDXL LoRA trainer", + "relative_path": "lora/pytorch_lora_weights.safetensors", + "bytes": 10910072, + "sha256": "03dd1968d37e7fdb5f1d86a5ad3ac09a961acd37b39353aed27f398e6266f785", + "rank": 4, + "alpha": 4, + "enabled": true + }, + "negative_prompt": "text, letters, words, captions, subtitles, watermark, logo, extra limbs, malformed hands, duplicate people, merged bodies, cropped subjects, distorted furniture, blurry, low resolution, oversaturated", + "cases": [ + { + "id": "obj_apple_01", + "category": "single_object", + "scene": "a plain warm classroom-like background with a small table", + "subject": "one red apple", + "count": "exactly one apple and no people", + "action": "no action", + "relation": "the apple is centered on the table", + "prompt": "one red apple on a small table, centered and fully visible", + "minimal": "one red apple only", + "seeds": [260101, 260201], + "confirmation_seeds": [260301], + "must_satisfy": ["exactly one apple", "apple fully visible", "no written text"], + "severe_failure": ["missing apple", "multiple apples", "text artifact"] + }, + { + "id": "action_wave_01", + "category": "person_action", + "scene": "a plain warm classroom-like background", + "subject": "one child", + "count": "exactly one person", + "action": "the child waves with the right hand while standing", + "relation": "the full body is visible and separated from the background", + "prompt": "exactly one child waves with the right hand while standing, fully visible", + "minimal": "one child waving", + "seeds": [260104, 260204], + "confirmation_seeds": [260304], + "must_satisfy": ["exactly one person", "one raised waving hand", "full body visible"], + "severe_failure": ["wrong count", "wrong action", "anatomy defect"] + }, + { + "id": "count_three_01", + "category": "person_count", + "scene": "a plain warm classroom-like background", + "subject": "three children", + "count": "exactly three people and no more", + "action": "standing still", + "relation": "the three children stand side by side with separated bodies", + "prompt": "exactly three children stand in one row, all three fully visible and separated", + "minimal": "three students only", + "seeds": [260109, 260209], + "confirmation_seeds": [260309], + "must_satisfy": ["exactly three people", "all visible", "no merged bodies"], + "severe_failure": ["wrong count", "merged people", "cropped subject"] + }, + { + "id": "class_pair_01", + "category": "classroom_activity", + "scene": "a simple classroom with an empty board and a table", + "subject": "two students", + "count": "exactly two people", + "action": "the students read one open book together", + "relation": "both students sit at the same table and share the book", + "prompt": "exactly two students sit at a classroom table and read one open book together, no writing on the board", + "minimal": "two students reading together", + "seeds": [260115, 260215], + "confirmation_seeds": [260315], + "must_satisfy": ["exactly two people", "one shared open book", "classroom interaction", "no board text"], + "severe_failure": ["wrong count", "missing book", "wrong scene", "text artifact"] + } + ], + "prompt_profiles": [ + { + "id": "P0", + "name": "current-baseline", + "description": "The current SSD-1B plus original LoRA evaluation profile, unchanged.", + "template": "flatcartoonstyle, clean flat two-dimensional educational cartoon, clear visual hierarchy, simple natural shapes, balanced composition, soft controlled colors, clear separation between people and objects, plain warm background, no written text, {prompt}" + }, + { + "id": "P1", + "name": "hard-constraint-first", + "description": "Count/action/relation constraints precede a short style tail.", + "template": "{hard_constraints}. {prompt}. Flat two-dimensional educational cartoon, simple shapes, clear separated subjects, plain warm background, no written text." + }, + { + "id": "P2", + "name": "natural-language-teaching-scene", + "description": "A teacher-like scene description with restrained style language.", + "template": "A clear classroom teaching illustration shows {natural_scene}. Use simple flat cartoon shapes, clear silhouettes, calm colors, and no written text." + }, + { + "id": "P3", + "name": "structured-fields", + "description": "The same intent is serialized as bounded subject/count/action/relation/scene/style/exclusions fields.", + "template": "subject: {subject}; count: {count}; action: {action}; relation: {relation}; scene: {scene}; style: flat two-dimensional educational cartoon with clear silhouettes; exclusions: no extra people, no text, no watermark" + }, + { + "id": "P4", + "name": "minimal-constraint", + "description": "Only the core teaching object/action and a few style words remain.", + "template": "{minimal}. simple flat cartoon, clear shapes, no text" + } + ], + "parameter_profiles": [ + { + "id": "A0", + "name": "current-baseline", + "description": "Current SSD-1B pilot settings.", + "scheduler": "euler", + "scheduler_class": "EulerDiscreteScheduler", + "steps": 8, + "guidance_scale": 7.5, + "width": 512, + "height": 512 + }, + { + "id": "A1", + "name": "balanced", + "description": "More steps with lower guidance for steadier semantics.", + "scheduler": "dpm++", + "scheduler_class": "DPMSolverMultistepScheduler", + "steps": 12, + "guidance_scale": 6.5, + "width": 512, + "height": 512 + }, + { + "id": "A2", + "name": "strong-constraint", + "description": "Ancestral Euler with slightly higher guidance, bounded below over-guidance failure.", + "scheduler": "euler_a", + "scheduler_class": "EulerAncestralDiscreteScheduler", + "steps": 10, + "guidance_scale": 8.5, + "width": 512, + "height": 512 + }, + { + "id": "A3", + "name": "soft", + "description": "DDIM with lower guidance to test whether extra elements and anatomy defects reduce.", + "scheduler": "ddim", + "scheduler_class": "DDIMScheduler", + "steps": 10, + "guidance_scale": 6.0, + "width": 512, + "height": 512 + } + ], + "stages": { + "prompt_only": { + "fixed_parameter_profile": "A0", + "prompt_profiles": ["P0", "P1", "P2", "P3", "P4"], + "cases": "all", + "seeds": "case.seeds" + }, + "parameter_only": { + "fixed_prompt_profile": "P0", + "parameter_profiles": ["A0", "A1", "A2", "A3"], + "cases": "all", + "seeds": "case.seeds" + }, + "top_combo_confirmation": { + "selection_limit": 3, + "seeds": "case.confirmation_seeds", + "selection_rule": "Select only after prompt-only and parameter-only review; record the evidence and keep teacher fields pending." + } + }, + "review_contract": { + "fields": ["better_than_baseline", "teaching_usable", "wrong_count", "wrong_action", "wrong_spatial_relation", "extra_person", "missing_object", "style_acceptable", "reviewer_notes"], + "automatic_teacher_scores": false, + "technical_observations_are_not_teacher_conclusions": true + } +} diff --git a/benchmarks/phase2c3/ablation_contract.py b/benchmarks/phase2c3/ablation_contract.py new file mode 100644 index 0000000..b12d9b6 --- /dev/null +++ b/benchmarks/phase2c3/ablation_contract.py @@ -0,0 +1,554 @@ +"""Pure contract and resumable artifact layer for the Phase 2C.3 ablation.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import os +import shutil +import struct +import zlib +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +SCHEMA = "hanclassstudio.phase2c3_prompt_parameter_ablation.v1" +STATE_SCHEMA = "hanclassstudio.phase2c3_prompt_parameter_ablation_state.v1" +REVIEW_SCHEMA = "hanclassstudio.phase2c3_prompt_parameter_ablation_review.v1" +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +_MAX_PNG_BYTES = 32 * 1024 * 1024 + + +class AblationError(RuntimeError): + pass + + +def canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def sha256_json(value: Any) -> str: + return sha256_bytes(canonical(value)) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise AblationError(f"invalid JSON: {path}") from exc + + +def write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_bytes(canonical(value) + b"\n") + os.replace(temporary, path) + + +def load_spec(path: Path) -> dict[str, Any]: + spec = read_json(path) + if not isinstance(spec, dict) or spec.get("schema") != SCHEMA: + raise AblationError("ablation schema mismatch") + if len(spec.get("cases", [])) != 4: + raise AblationError("the controlled ablation must contain exactly four cases") + if [p.get("id") for p in spec.get("prompt_profiles", [])] != ["P0", "P1", "P2", "P3", "P4"]: + raise AblationError("prompt profiles must be P0 through P4") + if [p.get("id") for p in spec.get("parameter_profiles", [])] != ["A0", "A1", "A2", "A3"]: + raise AblationError("parameter profiles must be A0 through A3") + if spec.get("stages", {}).get("prompt_only", {}).get("fixed_parameter_profile") != "A0": + raise AblationError("prompt-only stage must hold A0 fixed") + if spec.get("stages", {}).get("parameter_only", {}).get("fixed_prompt_profile") != "P0": + raise AblationError("parameter-only stage must hold P0 fixed") + return spec + + +def _by_id(items: Iterable[dict[str, Any]], item_id: str) -> dict[str, Any]: + for item in items: + if item.get("id") == item_id: + return item + raise AblationError(f"unknown profile or case: {item_id}") + + +def _hard_constraints(case: dict[str, Any]) -> str: + if case["id"] == "obj_apple_01": + return "ONE RED APPLE ONLY; ONE OBJECT; NO PEOPLE; NO EXTRA OBJECTS" + if case["id"] == "action_wave_01": + return "ONE PERSON ONLY; ONE CHILD; RIGHT HAND WAVING; NO EXTRA PEOPLE; NO CROWD" + if case["id"] == "count_three_01": + return "EXACTLY THREE PEOPLE; THREE CHILDREN ONLY; NO EXTRA PEOPLE; NO CROWD" + return "EXACTLY TWO PEOPLE; TWO STUDENTS ONLY; ONE SHARED BOOK; NO EXTRA PEOPLE" + + +def render_prompt(profile: dict[str, Any], case: dict[str, Any]) -> str: + template = profile["template"] + values = { + "prompt": case["prompt"], + "hard_constraints": _hard_constraints(case), + "natural_scene": f"{case['subject']} {case['action']}; {case['relation']} in {case['scene']}", + "subject": case["subject"], + "count": case["count"], + "action": case["action"], + "relation": case["relation"], + "scene": case["scene"], + "minimal": case["minimal"], + } + try: + return template.format(**values) + except KeyError as exc: + raise AblationError(f"prompt profile has an unknown field: {exc}") from exc + + +@dataclass(frozen=True) +class AblationTask: + key: str + stage: str + prompt_profile_id: str + parameter_profile_id: str + case_id: str + seed: int + request: dict[str, Any] + plan: dict[str, Any] + + +def build_tasks( + spec: dict[str, Any], + stage: str, + *, + combos: list[tuple[str, str]] | None = None, +) -> list[AblationTask]: + cases = spec["cases"] + profiles = {p["id"]: p for p in spec["prompt_profiles"]} + parameters = {p["id"]: p for p in spec["parameter_profiles"]} + if stage == "prompt-only": + pairs = [(pid, "A0") for pid in ["P0", "P1", "P2", "P3", "P4"]] + seeds_key = "seeds" + elif stage == "parameter-only": + pairs = [("P0", aid) for aid in ["A0", "A1", "A2", "A3"]] + seeds_key = "seeds" + elif stage == "top-combo-confirmation": + pairs = combos or [] + seeds_key = "confirmation_seeds" + if not pairs or len(pairs) > 3: + raise AblationError("confirmation needs one to three explicit prompt:parameter combos") + else: + raise AblationError(f"unknown stage: {stage}") + tasks: list[AblationTask] = [] + for prompt_id, parameter_id in pairs: + if prompt_id not in profiles or parameter_id not in parameters: + raise AblationError(f"unknown prompt/parameter pair: {prompt_id}:{parameter_id}") + for case in cases: + for seed in case[seeds_key]: + prompt = render_prompt(profiles[prompt_id], case) + task_key = f"{stage}:{prompt_id}:{parameter_id}:{case['id']}:{seed}" + request = { + "schema": "hanclassstudio.teaching_image_request.v1", + "asset_id": f"eval-phase2c3-{stage}-{prompt_id}-{parameter_id}-{case['id']}-{seed}", + "purpose": "teaching_illustration", + "subject": case["subject"], + "action": case["action"], + "environment": case["scene"], + "aspect_ratio": "1:1", + "seed": int(seed), + "source_trace": [ + "benchmark:phase2c3-prompt-parameter-ablation", + f"stage:{stage}", + f"prompt-profile:{prompt_id}", + f"parameter-profile:{parameter_id}", + f"case:{case['id']}", + f"seed:{seed}", + ], + } + plan = { + "schema": "hanclassstudio.phase2c3_execution_plan.v1", + "task_id": task_key, + "request_sha256": sha256_json(request), + "stage": stage, + "prompt_profile_id": prompt_id, + "parameter_profile_id": parameter_id, + "case_id": case["id"], + "model_revision": spec["model"]["revision"], + "positive_prompt": prompt, + "negative_prompt": spec["negative_prompt"], + "seed": int(seed), + "steps": parameters[parameter_id]["steps"], + "guidance_scale": parameters[parameter_id]["guidance_scale"], + "scheduler": parameters[parameter_id]["scheduler"], + "scheduler_class": parameters[parameter_id]["scheduler_class"], + "width": parameters[parameter_id]["width"], + "height": parameters[parameter_id]["height"], + "source_trace": request["source_trace"], + } + plan["execution_plan_sha256"] = sha256_json(plan) + tasks.append(AblationTask(task_key, stage, prompt_id, parameter_id, case["id"], int(seed), request, plan)) + return tasks + + +def verify_png(payload: bytes, *, width: int, height: int) -> dict[str, Any]: + if len(payload) <= 33 or len(payload) > _MAX_PNG_BYTES or not payload.startswith(_PNG_SIGNATURE): + raise AblationError("PNG signature or size is invalid") + offset = len(_PNG_SIGNATURE) + seen_ihdr = seen_idat = seen_iend = False + chunks = 0 + while offset + 12 <= len(payload): + length = struct.unpack(">I", payload[offset : offset + 4])[0] + data_start = offset + 8 + data_end = data_start + length + crc_end = data_end + 4 + if crc_end > len(payload): + raise AblationError("PNG chunk exceeds payload") + name = payload[offset + 4 : offset + 8] + data = payload[data_start:data_end] + expected_crc = struct.unpack(">I", payload[data_end:crc_end])[0] + if zlib.crc32(name + data) & 0xFFFFFFFF != expected_crc: + raise AblationError("PNG CRC mismatch") + chunks += 1 + if name == b"IHDR": + if seen_ihdr or len(data) != 13: + raise AblationError("PNG IHDR is invalid") + seen_ihdr = True + actual_width, actual_height = struct.unpack(">II", data[:8]) + if (actual_width, actual_height) != (width, height): + raise AblationError("PNG dimensions differ from fixed experiment") + elif name == b"IDAT": + seen_idat = True + elif name == b"IEND": + seen_iend = True + if crc_end != len(payload): + raise AblationError("PNG contains trailing bytes") + break + offset = crc_end + if not (seen_ihdr and seen_idat and seen_iend): + raise AblationError("PNG required chunks are missing") + return {"width": width, "height": height, "size_bytes": len(payload), "sha256": sha256_bytes(payload), "chunk_count": chunks} + + +def _atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_bytes(payload) + os.replace(temporary, path) + + +class AblationRunner: + """One stage state file; failed tasks do not stop later cases.""" + + def __init__(self, spec: dict[str, Any], output_dir: Path, state_path: Path, identity: dict[str, Any], tasks: list[AblationTask]) -> None: + self.spec = spec + self.output_dir = output_dir + self.state_path = state_path + self.identity = identity + self.tasks = tasks + self.state = self._load_or_create() + + def _new_state(self) -> dict[str, Any]: + return { + "schema": STATE_SCHEMA, + "experiment_id": self.spec["experiment_id"], + "stage": self.tasks[0].stage if self.tasks else "unknown", + "spec_sha256": sha256_json(self.spec), + "identity": self.identity, + "identity_sha256": sha256_json(self.identity), + "status": "pending", + "created_at": utc_now(), + "updated_at": utc_now(), + "tasks": { + task.key: { + "task": task.key, + "stage": task.stage, + "prompt_profile_id": task.prompt_profile_id, + "parameter_profile_id": task.parameter_profile_id, + "case_id": task.case_id, + "seed": task.seed, + "request": task.request, + "plan": task.plan, + "status": "pending", + "attempts": 0, + "result": None, + "failure": None, + } + for task in self.tasks + }, + } + + def _load_or_create(self) -> dict[str, Any]: + if not self.state_path.exists(): + state = self._new_state() + write_json(self.state_path, state) + return state + state = read_json(self.state_path) + if state.get("schema") != STATE_SCHEMA or state.get("spec_sha256") != sha256_json(self.spec): + raise AblationError("state belongs to a different ablation spec") + if state.get("identity_sha256") != sha256_json(self.identity): + invalidated = dict(state) + invalidated["status"] = "invalidated" + invalidated["invalidated_at"] = utc_now() + invalidated["invalidated_reason"] = "model/runtime/LoRA identity changed" + write_json(self.state_path.with_suffix(".invalidated.json"), invalidated) + state = self._new_state() + state["invalidated_previous_state"] = True + write_json(self.state_path, state) + return state + + def _save(self) -> None: + self.state["updated_at"] = utc_now() + write_json(self.state_path, self.state) + + def _artifact_valid(self, record: dict[str, Any]) -> bool: + result = record.get("result") or {} + image = self.output_dir / result.get("image_path", "") + provenance = self.output_dir / result.get("provenance_path", "") + return ( + image.is_file() + and provenance.is_file() + and sha256_file(image) == result.get("sha256") + and sha256_file(provenance) == result.get("provenance_sha256") + and result.get("identity_sha256") == sha256_json(self.identity) + ) + + def _write_manifest(self, result: dict[str, Any], task: AblationTask, provenance: dict[str, Any]) -> None: + path = self.output_dir / "asset_manifest.json" + manifest = read_json(path) if path.exists() else {"schema": "hanclassstudio.asset_manifest.v1", "images": []} + manifest["images"] = [item for item in manifest.get("images", []) if item.get("id") != result["asset_id"]] + manifest["images"].append({ + "id": result["asset_id"], + "kind": "image", + "path": result["image_path"], + "placeholder": False, + "mime_type": "image/png", + "content_hash": result["sha256"], + "review_state": "pending_review", + "request_fingerprint": result["request_sha256"], + "generation": { + "provider": "hcs.phase2c3.diffusers-ablation", + "model": self.spec["model"]["id"], + "lora": self.spec["lora"]["id"], + "prompt_profile_id": task.prompt_profile_id, + "parameter_profile_id": task.parameter_profile_id, + "prompt": task.plan["positive_prompt"], + "seed": task.seed, + "source_trace": task.request["source_trace"], + }, + "verified_image_artifact": { + "schema": "hanclassstudio.phase2c3_verified_image_artifact.v1", + "artifact_id": result["artifact_id"], + "asset_id": result["asset_id"], + "path": result["image_path"], + "mime_type": "image/png", + "width": result["width"], + "height": result["height"], + "size_bytes": result["size_bytes"], + "sha256": result["sha256"], + "provenance_ref": result["provenance_path"], + "provenance_sha256": result["provenance_sha256"], + "provenance": provenance, + }, + }) + write_json(path, manifest) + + def _persist(self, task: AblationTask, rendered: dict[str, Any]) -> dict[str, Any]: + payload = rendered.get("png_bytes") + if not isinstance(payload, bytes): + raise AblationError("executor did not return PNG bytes") + technical = rendered.get("technical") or {} + if technical.get("near_solid_warning"): + raise AblationError("technical precheck rejected a near-solid image") + verified = verify_png(payload, width=task.plan["width"], height=task.plan["height"]) + image_path = self.output_dir / "images" / task.stage / task.prompt_profile_id / task.parameter_profile_id / task.case_id / f"{task.seed}.png" + provenance_path = self.output_dir / "provenance" / f"{task.stage}-{task.prompt_profile_id}-{task.parameter_profile_id}-{task.case_id}-{task.seed}.json" + provenance = { + "schema": "hanclassstudio.phase2c3_provenance.v1", + "request": task.request, + "plan": task.plan, + "identity": self.identity, + "technical": {**verified, **technical}, + "source_trace": task.request["source_trace"], + "generated_at": utc_now(), + } + provenance_bytes = canonical(provenance) + b"\n" + _atomic_write(image_path, payload) + _atomic_write(provenance_path, provenance_bytes) + artifact_seed = f"{task.key}:{verified['sha256']}".encode() + artifact_id = f"img-{sha256_bytes(artifact_seed)[:24]}" + result = { + "task": task.key, + "request_sha256": task.plan["request_sha256"], + "execution_plan_sha256": task.plan["execution_plan_sha256"], + "artifact_id": artifact_id, + "asset_id": task.request["asset_id"], + "image_path": image_path.relative_to(self.output_dir).as_posix(), + "provenance_path": provenance_path.relative_to(self.output_dir).as_posix(), + "sha256": verified["sha256"], + "size_bytes": verified["size_bytes"], + "width": verified["width"], + "height": verified["height"], + "provenance_sha256": sha256_bytes(provenance_bytes), + "identity_sha256": sha256_json(self.identity), + "technical": provenance["technical"], + "review_status": "pending_review", + "failure_tags": [], + } + self._write_manifest(result, task, provenance) + return result + + def run(self, executor: Callable[[AblationTask], dict[str, Any]], *, max_attempts: int = 1, stop_after: int | None = None) -> dict[str, Any]: + attempted = 0 + self.state["status"] = "running" + self._save() + try: + for task in self.tasks: + record = self.state["tasks"][task.key] + if record["status"] == "succeeded" and self._artifact_valid(record): + continue + if record["status"] == "succeeded": + record["status"] = "pending" + record["result"] = None + while record["attempts"] < max_attempts: + record["attempts"] += 1 + attempted += 1 + try: + record["result"] = self._persist(task, executor(task)) + record["status"] = "succeeded" + record["failure"] = None + self._save() + break + except Exception as exc: # noqa: BLE001 - isolate one case + record["status"] = "failed" + record["failure"] = {"type": type(exc).__name__, "message": str(exc), "at": utc_now()} + self._save() + if stop_after is not None and attempted >= stop_after: + self.state["status"] = "paused" + self._save() + return self.state + except KeyboardInterrupt: + self.state["status"] = "paused" + self._save() + return self.state + self.state["status"] = "completed" if all(item["status"] == "succeeded" for item in self.state["tasks"].values()) else "completed_with_failures" + self._save() + return self.state + + +def _state_files(output_dir: Path) -> list[Path]: + return ( + sorted(path for path in (output_dir / "state").glob("*.json") if not path.name.endswith(".invalidated.json")) + if (output_dir / "state").is_dir() + else [] + ) + + +def all_records(output_dir: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for path in _state_files(output_dir): + state = read_json(path) + records.extend(state.get("tasks", {}).values()) + return records + + +def aggregate_report(spec: dict[str, Any], output_dir: Path, *, selection: dict[str, Any] | None = None) -> dict[str, Any]: + records = all_records(output_dir) + report: dict[str, Any] = { + "schema": "hanclassstudio.phase2c3_prompt_parameter_ablation_report.v1", + "experiment_id": spec["experiment_id"], + "spec_sha256": sha256_json(spec), + "total": len(records), + "succeeded": sum(r.get("status") == "succeeded" for r in records), + "failed": sum(r.get("status") == "failed" for r in records), + "pending": sum(r.get("status") not in {"succeeded", "failed"} for r in records), + "teacher_review": {"pending": sum(r.get("status") == "succeeded" for r in records), "reviewed": 0, "automatic_scores": False, "conclusion": None}, + "prompt_profiles": {}, + "parameter_profiles": {}, + "stages": {}, + "selection": selection, + "limitations": ["Technical checks and engineering observations are not teacher conclusions.", "No automatic better_than_baseline or teaching_usable values are produced."], + } + for record in records: + stage = record.get("stage") + report["stages"].setdefault(stage, {"total": 0, "succeeded": 0, "failed": 0}) + report["stages"][stage]["total"] += 1 + report["stages"][stage][record.get("status")] = report["stages"][stage].get(record.get("status"), 0) + 1 + if record.get("status") != "succeeded" or not record.get("result"): + continue + result = record["result"] + technical = result.get("technical", {}) + for key, value in (("prompt_profiles", record.get("prompt_profile_id")), ("parameter_profiles", record.get("parameter_profile_id"))): + bucket = report[key].setdefault(value, {"total": 0, "succeeded": 0, "durations": [], "near_solid_warnings": 0}) + bucket["total"] += 1 + bucket["succeeded"] += 1 + if technical.get("duration_seconds") is not None: + bucket["durations"].append(technical["duration_seconds"]) + bucket["near_solid_warnings"] += int(bool(technical.get("near_solid_warning"))) + for bucket_group in (report["prompt_profiles"], report["parameter_profiles"]): + for bucket in bucket_group.values(): + durations = bucket.pop("durations") + bucket["mean_duration_seconds"] = sum(durations) / len(durations) if durations else None + write_json(output_dir / "ablation-report.json", report) + return report + + +def write_review_package(spec: dict[str, Any], output_dir: Path) -> Path: + package = output_dir / "review-package" + images_dir = package / "images" + images_dir.mkdir(parents=True, exist_ok=True) + fields = ["stage", "prompt_profile_id", "parameter_profile_id", "case_id", "seed", "asset_id", "artifact_id", "review_status", "better_than_baseline", "teaching_usable", "wrong_count", "wrong_action", "wrong_spatial_relation", "extra_person", "missing_object", "style_acceptable", "reviewer_notes"] + reviews: list[dict[str, Any]] = [] + manifest: list[dict[str, Any]] = [] + for record in all_records(output_dir): + if record.get("status") != "succeeded" or not record.get("result"): + continue + result = record["result"] + source = output_dir / result["image_path"] + name = f"{record['stage']}-{record['prompt_profile_id']}-{record['parameter_profile_id']}-{record['case_id']}-{record['seed']}.png" + if source.is_file(): + shutil.copy2(source, images_dir / name) + review = { + "schema": REVIEW_SCHEMA, + "stage": record["stage"], + "prompt_profile_id": record["prompt_profile_id"], + "parameter_profile_id": record["parameter_profile_id"], + "case_id": record["case_id"], + "seed": record["seed"], + "asset_id": result["asset_id"], + "artifact_id": result["artifact_id"], + "image_path": f"images/{name}", + "review_status": "pending_review", + **{field: None for field in fields[8:-1]}, + "reviewer_notes": "", + } + reviews.append(review) + manifest.append({"task": record["task"], "result": result, "request": record["request"], "plan": record["plan"]}) + write_json(package / "teacher-reviews.pending.json", {"schema": REVIEW_SCHEMA, "reviews": reviews}) + with (package / "teacher-reviews.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=fields) + writer.writeheader() + for review in reviews: + writer.writerow({key: json.dumps(review[key], ensure_ascii=False) if isinstance(review[key], list) else review[key] for key in fields}) + write_json(package / "experiment-manifest.json", {"schema": SCHEMA, "spec": spec, "records": manifest}) + cards = [] + for review in reviews: + cards.append( + f"

{review['stage']} / {review['prompt_profile_id']} / {review['parameter_profile_id']} / {review['case_id']} / seed {review['seed']}

" + f"pending teacher review

{review['artifact_id']}

" + ) + html = "Phase 2C.3 prompt and parameter ablation

Phase 2C.3 ablation

Technical checks are separate from teacher review. Every review field is pending.

" + "".join(cards) + "
\n" + (package / "comparison.html").write_text(html, encoding="utf-8") + (package / "README.md").write_text("# Phase 2C.3 prompt/parameter ablation review package\n\nOpen `comparison.html`; teacher fields remain `pending_review`.\n", encoding="utf-8") + return package diff --git a/benchmarks/phase2c3/run_ablation.py b/benchmarks/phase2c3/run_ablation.py new file mode 100644 index 0000000..2b27d02 --- /dev/null +++ b/benchmarks/phase2c3/run_ablation.py @@ -0,0 +1,227 @@ +"""Real, bounded SSD-1B prompt/parameter ablation runner. + +This is an opt-in evaluation executor. It never edits the production ComfyUI +Model Package or Workflow Pack and accepts no caller-supplied graph. +""" + +from __future__ import annotations + +import argparse +import io +import json +import subprocess +import time +from pathlib import Path +from typing import Any + +import psutil +import torch +from ablation_contract import ( + AblationError, + AblationRunner, + AblationTask, + aggregate_report, + build_tasks, + load_spec, + read_json, + sha256_file, + write_review_package, +) +from diffusers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + StableDiffusionXLPipeline, +) + +SCHEDULERS = { + "euler": EulerDiscreteScheduler, + "euler_a": EulerAncestralDiscreteScheduler, + "dpm++": DPMSolverMultistepScheduler, + "ddim": DDIMScheduler, +} + + +def _swap_usage() -> str: + try: + return subprocess.check_output(["sysctl", "-n", "vm.swapusage"], text=True, timeout=2).strip() + except (OSError, subprocess.SubprocessError): + return "unavailable" + + +def _mps_memory() -> dict[str, int | None]: + if not hasattr(torch, "mps") or not torch.backends.mps.is_available(): + return {"allocated_bytes": None, "driver_allocated_bytes": None} + try: + return { + "allocated_bytes": int(torch.mps.current_allocated_memory()), + "driver_allocated_bytes": int(torch.mps.driver_allocated_memory()), + } + except RuntimeError: + return {"allocated_bytes": None, "driver_allocated_bytes": None} + + +def _identity(spec: dict[str, Any], model_dir: Path, lora_file: Path) -> dict[str, Any]: + files: dict[str, dict[str, Any]] = {} + for item in spec["model"]["selected_files"]: + path = model_dir / item["path"] + if not path.is_file() or path.stat().st_size != item["bytes"] or sha256_file(path) != item["sha256"]: + raise AblationError(f"fixed SSD model file identity mismatch: {path}") + files[item["path"]] = {"bytes": path.stat().st_size, "sha256": item["sha256"]} + lora = spec["lora"] + if not lora_file.is_file() or lora_file.stat().st_size != lora["bytes"] or sha256_file(lora_file) != lora["sha256"]: + raise AblationError(f"fixed LoRA identity mismatch: {lora_file}") + return { + "backend": "huggingface-diffusers", + "diffusers_revision": spec["runtime_contract"]["diffusers_revision"], + "torch_version": torch.__version__, + "device": "mps", + "host_contract": spec["host_contract"], + "model_id": spec["model"]["id"], + "model_revision": spec["model"]["revision"], + "model_files": files, + "lora_id": lora["id"], + "lora_bytes": lora_file.stat().st_size, + "lora_sha256": lora["sha256"], + "vae_dtype": "float32", + } + + +class SSD1BExecutor: + def __init__(self, model_dir: Path, lora_file: Path, *, width: int, height: int) -> None: + self.process = psutil.Process() + self.width = width + self.height = height + started = time.monotonic() + self.pipeline = StableDiffusionXLPipeline.from_pretrained( + model_dir.resolve(), + torch_dtype=torch.float16, + variant="fp16", + use_safetensors=True, + local_files_only=True, + ) + self.pipeline.to("mps") + # MPS fp16 VAE decode can overflow for SDXL-family checkpoints. + self.pipeline.vae.to(dtype=torch.float32) + self.pipeline.enable_attention_slicing() + self.pipeline.set_progress_bar_config(disable=True) + self.load_seconds = time.monotonic() - started + self.load_rss = self.process.memory_info().rss + self.load_mps = _mps_memory() + self.pipeline.load_lora_weights(lora_file.parent, weight_name=lora_file.name) + self.lora_load_seconds = time.monotonic() - started - self.load_seconds + self._scheduler_name: str | None = None + + def _set_scheduler(self, name: str) -> None: + if self._scheduler_name == name: + return + scheduler_class = SCHEDULERS.get(name) + if scheduler_class is None: + raise AblationError(f"unsupported scheduler profile: {name}") + config = dict(self.pipeline.scheduler.config) + if name == "dpm++": + config.update({"algorithm_type": "dpmsolver++", "solver_order": 2, "lower_order_final": True}) + self.pipeline.scheduler = scheduler_class.from_config(config) + self._scheduler_name = name + + def __call__(self, task: AblationTask) -> dict[str, Any]: + self._set_scheduler(task.plan["scheduler"]) + before_rss = self.process.memory_info().rss + before_mps = _mps_memory() + started = time.monotonic() + generator = torch.Generator(device="cpu").manual_seed(task.seed) + with torch.inference_mode(): + output = self.pipeline( + prompt=task.plan["positive_prompt"], + negative_prompt=task.plan["negative_prompt"], + width=self.width, + height=self.height, + num_inference_steps=task.plan["steps"], + guidance_scale=task.plan["guidance_scale"], + generator=generator, + ) + image = output.images[0] + buffer = io.BytesIO() + image.save(buffer, format="PNG", optimize=True) + payload = buffer.getvalue() + extrema = image.convert("RGB").getextrema() + near_solid = all((high - low) <= 4 for low, high in extrema) + ended = time.monotonic() + return { + "png_bytes": payload, + "technical": { + "duration_seconds": round(ended - started, 3), + "load_seconds": round(self.load_seconds, 3), + "lora_load_seconds": round(self.lora_load_seconds, 3), + "rss_before_bytes": before_rss, + "rss_after_bytes": self.process.memory_info().rss, + "rss_load_bytes": self.load_rss, + "mps_before": before_mps, + "mps_after": _mps_memory(), + "mps_load": self.load_mps, + "swap_usage": _swap_usage(), + "near_solid_warning": near_solid, + "device": "mps", + "scheduler_profile": task.plan["scheduler"], + "steps": task.plan["steps"], + "guidance_scale": task.plan["guidance_scale"], + }, + } + + +def _args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c3/ablation.v1.json")) + parser.add_argument("--model-dir", type=Path, default=Path("runtime/phase2c3-prompt-parameter-ablation/model")) + parser.add_argument("--lora-file", type=Path, default=Path("runtime/phase2c3-prompt-parameter-ablation/lora/pytorch_lora_weights.safetensors")) + parser.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c3-prompt-parameter-ablation")) + parser.add_argument("--stage", choices=["prompt-only", "parameter-only", "top-combo-confirmation"]) + parser.add_argument("--combos", help="Comma-separated prompt:parameter pairs for confirmation, at most three") + parser.add_argument("--max-attempts", type=int, default=1) + parser.add_argument("--stop-after", type=int) + parser.add_argument("--report-only", action="store_true") + parser.add_argument("--review-package", action="store_true") + parser.add_argument("--preview", action="store_true") + return parser.parse_args() + + +def main() -> None: + args = _args() + spec = load_spec(args.spec) + args.output_dir.mkdir(parents=True, exist_ok=True) + selection_path = args.output_dir / "selection.v1.json" + selection = read_json(selection_path) if selection_path.exists() else None + if args.report_only: + report = aggregate_report(spec, args.output_dir, selection=selection) + if args.review_package: + write_review_package(spec, args.output_dir) + print(json.dumps({"status": "report", "total": report["total"], "succeeded": report["succeeded"], "failed": report["failed"], "pending": report["pending"]}, ensure_ascii=False)) + return + if not args.stage: + raise SystemExit("--stage is required unless --report-only is used") + combos = None + if args.combos: + combos = [] + for raw in args.combos.split(","): + prompt_id, separator, parameter_id = raw.strip().partition(":") + if not separator: + raise SystemExit(f"invalid combo: {raw}") + combos.append((prompt_id, parameter_id)) + tasks = build_tasks(spec, args.stage, combos=combos) + if args.preview: + print(json.dumps({"stage": args.stage, "tasks": len(tasks), "first": tasks[0].plan if tasks else None}, ensure_ascii=False, indent=2)) + return + state_path = args.output_dir / "state" / f"{args.stage}.json" + identity = _identity(spec, args.model_dir, args.lora_file) + runner = AblationRunner(spec, args.output_dir, state_path, identity, tasks) + executor = SSD1BExecutor(args.model_dir, args.lora_file, width=spec["runtime_contract"]["width"], height=spec["runtime_contract"]["height"]) + state = runner.run(executor, max_attempts=max(1, args.max_attempts), stop_after=args.stop_after) + report = aggregate_report(spec, args.output_dir, selection=selection) + if args.review_package: + write_review_package(spec, args.output_dir) + print(json.dumps({"stage": args.stage, "status": state["status"], "tasks": len(tasks), "succeeded": report["succeeded"], "failed": report["failed"], "pending": report["pending"]}, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/phase2c3/selection.v1.json b/benchmarks/phase2c3/selection.v1.json new file mode 100644 index 0000000..2608176 --- /dev/null +++ b/benchmarks/phase2c3/selection.v1.json @@ -0,0 +1,23 @@ +{ + "schema": "hanclassstudio.phase2c3_ablation_selection.v1", + "experiment_id": "phase2c3-ssd1b-prompt-parameter-ablation", + "stage_1_prompt_observation": { + "provisional_local_candidate": "P1", + "reason": "P1 produced a plausible exact-three-person frame in one seed and sometimes reduced broad-scene extra people, but it still failed the one-person action and classroom-pair cases and did not stably satisfy exact count across both seeds.", + "stable_winner": false + }, + "stage_2_parameter_observation": { + "provisional_local_candidate": "A0", + "reason": "A0 retained the current baseline and was the only parameter group with an exact-three-person-looking frame in one inspected seed; A1/A2/A3 changed rendering or composition but did not consistently improve count/action/classroom relation.", + "stable_winner": false + }, + "confirmation": { + "status": "not_run", + "reason": "The protocol permits confirmation only when the first two layers show a clear improvement. Neither layer produced a stable semantic winner, so additional seeds would add samples without testing a justified candidate.", + "selected_combos": [] + }, + "teacher_review": { + "status": "pending_review", + "automatic_scores": false + } +} diff --git a/benchmarks/phase2c3/test_ablation_contract.py b/benchmarks/phase2c3/test_ablation_contract.py new file mode 100644 index 0000000..4c7eeb9 --- /dev/null +++ b/benchmarks/phase2c3/test_ablation_contract.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import json +import struct +import zlib +from pathlib import Path + +from ablation_contract import ( + AblationRunner, + aggregate_report, + build_tasks, + load_spec, + verify_png, + write_review_package, +) + +ROOT = Path(__file__).parents[2] +SPEC = ROOT / "benchmarks/phase2c3/ablation.v1.json" + + +def _png(width: int = 512, height: int = 512) -> bytes: + raw = b"".join(b"\x00" + b"\x80\x90\xa0" * width for _ in range(height)) + compressed = zlib.compress(raw) + + def chunk(name: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + name + data + struct.pack(">I", zlib.crc32(name + data) & 0xFFFFFFFF) + + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", compressed) + chunk(b"IEND", b"") + + +def test_fixed_matrix_and_profiles() -> None: + spec = load_spec(SPEC) + assert len(build_tasks(spec, "prompt-only")) == 40 + assert len(build_tasks(spec, "parameter-only")) == 32 + assert len(build_tasks(spec, "top-combo-confirmation", combos=[("P1", "A1"), ("P2", "A0"), ("P0", "A0")])) == 12 + p0 = build_tasks(spec, "prompt-only")[0].plan["positive_prompt"] + p1 = build_tasks(spec, "prompt-only")[8].plan["positive_prompt"] + assert p0.startswith("flatcartoonstyle,") + assert "ONE PERSON ONLY" in p1 or "ONE RED APPLE ONLY" in p1 + assert spec["parameter_profiles"][0]["scheduler_class"] == "EulerDiscreteScheduler" + + +def test_png_verifier_and_resumable_artifact_contract(tmp_path: Path) -> None: + spec = load_spec(SPEC) + tasks = build_tasks(spec, "top-combo-confirmation", combos=[("P0", "A0")])[:1] + output = tmp_path / "run" + state_path = output / "state" / "confirmation.json" + identity = {"model": "ssd-test", "lora": "lora-test", "runtime": "mps"} + payload = _png() + assert verify_png(payload, width=512, height=512)["width"] == 512 + calls: list[str] = [] + + def execute(task): + calls.append(task.key) + return {"png_bytes": payload, "technical": {"duration_seconds": 0.1, "near_solid_warning": False}} + + runner = AblationRunner(spec, output, state_path, identity, tasks) + assert runner.run(execute)["status"] == "completed" + assert len(calls) == 1 + runner = AblationRunner(spec, output, state_path, identity, tasks) + assert runner.run(execute)["status"] == "completed" + assert len(calls) == 1 + report = aggregate_report(spec, output) + assert report["succeeded"] == 1 + assert report["teacher_review"]["reviewed"] == 0 + package = write_review_package(spec, output) + reviews = json.loads((package / "teacher-reviews.pending.json").read_text()) + assert len(reviews["reviews"]) == 1 + assert reviews["reviews"][0]["better_than_baseline"] is None diff --git a/docs/phase2c3-prompt-parameter-ablation.md b/docs/phase2c3-prompt-parameter-ablation.md new file mode 100644 index 0000000..1eed1ef --- /dev/null +++ b/docs/phase2c3-prompt-parameter-ablation.md @@ -0,0 +1,165 @@ +# Phase 2C.3 SSD-1B prompt / parameter ablation + +This is a controlled evaluation of the existing SSD-1B plus original +flat-cartoon LoRA route. It does not change the production Model Package, +Workflow Pack, default prompt profile, sampler, scheduler, steps, Provider Hub, +or teacher UI. + +## Result first + +No prompt family or sampling parameter group produced a stable, clearly +visible improvement over the current SSD-1B + LoRA baseline. P1 (hard +constraints first) had a locally plausible exact-three-person frame in one +seed and sometimes fewer extra people in broad scenes, but still failed the +one-person wave and classroom-pair cases. A1/A2/A3 changed rendering and speed +without consistently improving exact count, action, or classroom relation. + +The evidence therefore supports the current working conclusion: + +> SSD-1B’s main bottleneck for these precise teaching semantics is still model +> semantic control, not an easily corrected prompt/profile or sampling setting. + +The experiment stops after 72 real images. Stage 3 confirmation was not run +because the protocol allows it only after the first two layers show a clear +stable improvement. No production prompt or parameter update is recommended. + +## Fixed identities and boundaries + +The complete machine-readable contract is +[`benchmarks/phase2c3/ablation.v1.json`](../benchmarks/phase2c3/ablation.v1.json). + +| Item | Fixed identity | +| --- | --- | +| Base model | `segmind/SSD-1B` @ `60987f37e94cd59c36b1cba832b9f97b57395a10` | +| Model license | Apache-2.0 model card; see the [official SSD-1B card](https://huggingface.co/segmind/SSD-1B) | +| Selected model bytes | 4,465,653,694 fp16 bytes; four component SHA-256 values are in the manifest | +| LoRA | existing project-authored `flat-cartoon-lora`, 10,910,072 bytes, SHA-256 `03dd1968d37e7fdb5f1d86a5ad3ac09a961acd37b39353aed27f398e6266f785` | +| Backend | Diffusers v0.36.0, MPS, VAE decode float32, denoiser/text encoders fp16 | +| Size | 512×512 for every task | +| Custom nodes / graph | none; no caller-supplied ComfyUI graph | +| Production mutation | none; this is an ignored, opt-in evaluation executor | + +The local model, LoRA, environment, generated PNGs, state, cache, and reports +are under `runtime/phase2c3-prompt-parameter-ablation/` and are not tracked by +Git. The runner reuses the fixed SSD identity and the existing original LoRA; +it does not download another model or LoRA. + +## Prompt profiles + +All five profiles use the same negative prompt and the same A0 parameters. Only +the positive prompt text changes. + +| ID | Definition | What it tests | +| --- | --- | --- | +| P0 | Current route: `flatcartoonstyle`, flat 2D educational cartoon, clear hierarchy/separation, warm background, no text, then the case prompt | Real baseline | +| P1 | Uppercase count/action/relation constraints first (`ONE PERSON ONLY`, `EXACTLY THREE PEOPLE`, `NO CROWD`, etc.), then the case prompt and short style tail | Whether explicit hard constraints improve count/action | +| P2 | Natural teacher-like sentence: “A clear classroom teaching illustration shows …” with restrained style language | Whether natural scene language is easier to follow | +| P3 | `subject; count; action; relation; scene; style; exclusions` fields serialized into one prompt | Whether structured wording helps | +| P4 | Core object/action only plus `simple flat cartoon, clear shapes, no text` | Whether less prompt interference improves control | + +The exact templates and each rendered prompt are recorded in every task plan +and provenance file. No negative prompt or seed changed during this layer. + +## Parameter profiles + +The parameter-only layer fixes P0 and varies only the following controlled +settings. These are concrete Diffusers scheduler classes, not arbitrary graph +names. + +| ID | Scheduler | Steps | Guidance | Purpose | +| --- | --- | ---: | ---: | --- | +| A0 | `EulerDiscreteScheduler` (`euler`) | 8 | 7.5 | Current SSD pilot baseline | +| A1 | `DPMSolverMultistepScheduler` (`dpm++`, algorithm `dpmsolver++`) | 12 | 6.5 | More steps, lower guidance / balanced | +| A2 | `EulerAncestralDiscreteScheduler` (`euler_a`) | 10 | 8.5 | Slightly stronger instruction pressure | +| A3 | `DDIMScheduler` (`ddim`) | 10 | 6.0 | Softer guidance / fewer extra elements | + +Dimensions, negative prompt, LoRA, cases, and seeds remained fixed. No +production sampler, scheduler, or step count was edited. + +## Experiment matrix and execution + +- Stage 1 prompt-only: P0/P1/P2/P3/P4 × A0 × four cases × two fixed seeds = + 40 images. +- Stage 2 parameter-only: P0 × A0/A1/A2/A3 × four cases × two fixed seeds = + 32 images. +- Stage 3 confirmation: at most three combinations × one extra seed per case; + not run because neither earlier layer produced a stable improvement. + +Cases were deliberately limited to `obj_apple_01`, `action_wave_01`, +`count_three_01`, and `class_pair_01`. Each task records a bounded +`TeachingImageRequest`-shaped request, compiled plan, model/LoRA identity, +profile IDs, case ID, seed, PNG SHA-256, duration, MPS/RSS/swap evidence, and +failure state. The runner persists state after every attempt, resumes only +when identity and artifact hashes match, isolates per-case failures, and +invalidates a stage if the fixed model/runtime/LoRA identity changes. + +The review export is +[`comparison.html`](). +The image files are in +[`review-package/images`](). + +## Technical results + +| Layer / group | Images | Success | Mean generation | Min–max | Near-solid outputs | +| --- | ---: | ---: | ---: | ---: | ---: | +| Prompt P0 | 8 | 8/8 | 11.238 s | 11.155–11.450 s | 0 | +| Prompt P1 | 8 | 8/8 | 12.878 s | 11.230–15.455 s | 0 | +| Prompt P2 | 8 | 8/8 | 15.895 s | 15.522–16.227 s | 0 | +| Prompt P3 | 8 | 8/8 | 15.736 s | 15.423–15.911 s | 0 | +| Prompt P4 | 8 | 8/8 | 16.137 s | 15.855–16.514 s | 0 | +| Parameter A0 | 8 | 8/8 | 12.036 s | 11.278–13.510 s | 0 | +| Parameter A1 | 8 | 8/8 | 20.839 s | 20.216–21.738 s | 0 | +| Parameter A2 | 8 | 8/8 | 22.628 s | 18.953–24.995 s | 0 | +| Parameter A3 | 8 | 8/8 | 19.648 s | 17.432–21.824 s | 0 | + +Across all 72 images: 72/72 technical success, 0 failures, 0 near-solid +warnings. The maximum observed MPS driver allocation was 8,259,387,392 bytes; +the maximum recorded post-image process RSS was 607,649,792 bytes. Swap usage +was recorded per image because the 16 GB host was under memory pressure. These +are observed-host measurements, not a claim that the route is comfortable on +every 16 GB machine. + +## Engineering observations, not teacher conclusions + +The contact sheets show the same pattern across both fixed seeds: + +- P0 is coherent but commonly produces two or more people for a one-person + wave, and misses the requested two-person classroom interaction. +- P1 can produce three visible people in the count case, but the other seed and + the action/classroom cases still miss exact count or action. It is a local + candidate, not a stable winner. +- P2/P3/P4 often change framing, background, or subject abstraction more than + they improve the requested relation. They do not reliably preserve exact + count or action. +- A0 retains the current baseline behavior. A1/A2/A3 do not fix the same + semantic failures; A2 visibly increases composition variability in inspected + frames, while all three cost materially more time than A0. +- The single-object case still often contains multiple or malformed object + forms. The exact three-person case and one-child wave remain unreliable. + +These observations are intentionally not written into `better_than_baseline`, +`teaching_usable`, or any teacher score. All 72 review records remain +`pending_review` in `teacher-reviews.pending.json` and `teacher-reviews.csv`. + +## Decision + +1. Do not update the production prompt profile, sampler, scheduler, steps, or + Model Package from this experiment. +2. Do not claim a “clearly better” prompt + parameter combination. The + provisional local pair P1/A0 is not confirmed and is not production-ready. +3. Treat the persistent exact-count, action, and relation errors as evidence + that model semantic capacity is the main current bottleneck. +4. If teacher review later finds value, limit any future prompt work to broad, + open-ended illustration scenes. Do not expose exact-count/action claims + without a stronger model or another controlled architecture. +5. A larger model comparison remains the next meaningful experiment; unlimited + prompt farming is not justified by this result. + +## Validation and remaining boundaries + +Focused contract tests, full API tests, TypeScript noEmit, Vite build, Web state +tests, Playwright, Ruff, and `git diff --check` are required before publishing +the Draft PR. No image, model, LoRA, cache, or runtime output is committed. +Teacher quality review is still outstanding, the test set is intentionally only +four cases and two seeds, and this loop does not compare a new base-only branch +because its controlled question is the current SSD-1B + original LoRA route.