diff --git a/apps/api/src/hcs_api/comfyui_teaching_image.py b/apps/api/src/hcs_api/comfyui_teaching_image.py index 2fdae37..bd990d8 100644 --- a/apps/api/src/hcs_api/comfyui_teaching_image.py +++ b/apps/api/src/hcs_api/comfyui_teaching_image.py @@ -605,6 +605,9 @@ def _execute_fixed_plan( {"filename": filename, "subfolder": "", "type": "output"} ) return _http_image(port, query, maximum_bytes), prompt_id + except KeyboardInterrupt: + _cancel_job_if_still_owned(plan, prompt_id) + raise except TeachingImageError: _cancel_job_if_still_owned(plan, prompt_id) raise diff --git a/apps/api/src/hcs_api/teaching_image_benchmark.py b/apps/api/src/hcs_api/teaching_image_benchmark.py new file mode 100644 index 0000000..ddbaa79 --- /dev/null +++ b/apps/api/src/hcs_api/teaching_image_benchmark.py @@ -0,0 +1,966 @@ +"""Reproducible, teacher-reviewable benchmark for the fixed Phase 2C image path.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import shutil +import sys +import time +import uuid +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from .comfyui_archive import load_runtime_manifest +from .comfyui_model import ( + WORKFLOW_PACK_SHA256, + ComfyUIModelError, + load_model_manifest, + load_workflow_pack, + model_installation_identity, + validate_model_installation, +) +from .comfyui_runtime import ( + ComfyUIRuntimeError, + runtime_installation_identity, + runtime_snapshot, +) +from .comfyui_teaching_image import ( + TeachingImageError, + TeachingImageRequest, + generate_teaching_image, + generation_capability_snapshot, + verify_png, +) +from .models import AssetManifest, TeachingImageProvenance, VerifiedImageArtifact + +BENCHMARK_SCHEMA = "hanclassstudio.teaching_image_benchmark.v1" +REVIEW_SCHEMA = "hanclassstudio.teacher_image_reviews.v1" +_MAX_JSON_BYTES = 32 * 1024 * 1024 +_REQUIRED_CATEGORIES = frozenset( + { + "single_object", + "person_action", + "person_count", + "spatial_relation", + "classroom_activity", + "daily_communication", + "emotion_expression", + "cultural_scene", + "event_sequence", + "hard_combination", + } +) +_FAILURE_LABELS = frozenset( + { + "wrong_count", + "wrong_action", + "wrong_scene", + "wrong_spatial_relation", + "missing_object", + "anatomy_defect", + "text_artifact", + "culturally_inappropriate", + "visually_confusing", + "not_teaching_usable", + } +) +_RATING_FIELDS = ( + "goal_relevance", + "instruction_following", + "person_object_count", + "action_accuracy", + "spatial_relation_accuracy", + "classroom_usability", + "visual_integrity", + "cultural_age_appropriateness", +) + + +class BenchmarkError(RuntimeError): + """A benchmark definition, state, or execution error.""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +class BenchmarkBlockedError(BenchmarkError): + """A fail-closed blocker that requires an external/runtime change.""" + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + +class BenchmarkFixedIdentity(_StrictModel): + model_package_id: str + model_version: str + model_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + workflow_pack_id: str + workflow_version: str + workflow_pack_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_id: Literal["comfyui"] + runtime_version: str + runtime_source_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + prompt_profile_id: str + + +class BenchmarkRequest(_StrictModel): + purpose: Literal["classroom_scene", "vocabulary_image", "teaching_illustration"] + subject: str = Field(min_length=1, max_length=240) + action: str = Field(min_length=1, max_length=240) + environment: str = Field(min_length=1, max_length=240) + aspect_ratio: Literal["1:1", "4:3", "16:9"] + + +class BenchmarkExpected(_StrictModel): + people: int = Field(ge=0, le=20) + objects: list[str] = Field(max_length=20) + actions: list[str] = Field(max_length=12) + relations: list[str] = Field(max_length=12) + scene: str = Field(min_length=1, max_length=240) + + +class BenchmarkCase(_StrictModel): + case_id: str = Field(pattern=r"^[a-z][a-z0-9_]{2,63}$") + category: Literal[ + "single_object", + "person_action", + "person_count", + "spatial_relation", + "classroom_activity", + "daily_communication", + "emotion_expression", + "cultural_scene", + "event_sequence", + "hard_combination", + ] + title: str = Field(min_length=1, max_length=120) + teaching_goal: str = Field(min_length=1, max_length=500) + prompt: str = Field(min_length=1, max_length=800) + negative_prompt: str = Field(min_length=1, max_length=1200) + request: BenchmarkRequest + seeds: list[int] = Field(min_length=1, max_length=2) + expected: BenchmarkExpected + must_satisfy: list[str] = Field(min_length=1, max_length=12) + severe_failures: list[str] = Field(min_length=1, max_length=12) + + @model_validator(mode="after") + def _prompt_is_the_request_intent(self) -> BenchmarkCase: + expected = ( + f"subject: {self.request.subject}; action: {self.request.action}; " + f"environment: {self.request.environment}" + ) + if self.prompt != expected: + raise ValueError("case prompt must be the canonical controlled request intent") + if len(set(self.seeds)) != len(self.seeds): + raise ValueError("case seeds must be unique") + return self + + +class BenchmarkSpec(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_benchmark.v1"] = Field( + default=BENCHMARK_SCHEMA, alias="schema" + ) + benchmark_id: str = Field(pattern=r"^[a-z][a-z0-9-]{2,63}$") + version: str = Field(pattern=r"^\d+\.\d+\.\d+$") + fixed_identity: BenchmarkFixedIdentity + negative_prompt: str = Field(min_length=1, max_length=1200) + cases: list[BenchmarkCase] = Field(min_length=20, max_length=30) + + @model_validator(mode="after") + def _case_contract(self) -> BenchmarkSpec: + case_ids = [case.case_id for case in self.cases] + if len(set(case_ids)) != len(case_ids): + raise ValueError("benchmark case ids must be unique") + if {case.category for case in self.cases} != _REQUIRED_CATEGORIES: + raise ValueError("benchmark must cover every required teaching category") + if any(case.negative_prompt != self.negative_prompt for case in self.cases): + raise ValueError("every case must record the fixed negative prompt") + return self + + +class BenchmarkObservedIdentity(BenchmarkFixedIdentity): + runtime_installation_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_process_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + runtime_port: int = Field(ge=1024, le=65535) + model_installation_identity: str = Field(pattern=r"^[0-9a-f]{64}$") + checked_at: str + + +class BenchmarkErrorRecord(_StrictModel): + code: str + message: str + attempt: int = Field(ge=1) + recoverable: bool = True + occurred_at: str + + +class BenchmarkTechnicalCheck(_StrictModel): + status: Literal["passed"] = "passed" + image_path: str + provenance_path: str + width: int + height: int + image_size_bytes: int + image_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + provenance_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + manifest_asset_id: str + request_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + execution_plan_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + checks: list[str] = Field(min_length=1) + + +class BenchmarkCaseResult(_StrictModel): + case_id: str + seed: int + status: Literal["pending", "running", "succeeded", "failed", "invalidated"] = "pending" + attempts: int = Field(default=0, ge=0) + started_at: str | None = None + completed_at: str | None = None + artifact: VerifiedImageArtifact | None = None + technical_checks: BenchmarkTechnicalCheck | None = None + error: BenchmarkErrorRecord | None = None + manual_review: None = None + + +class TeacherReviewRecord(_StrictModel): + schema_: Literal["hanclassstudio.teacher_image_review.v1"] = Field( + default="hanclassstudio.teacher_image_review.v1", alias="schema" + ) + case_key: str + case_id: str + seed: int + artifact_id: str + review_state: Literal["pending_review", "reviewed"] = "pending_review" + reviewer_id: str = "" + goal_relevance: int | None = Field(default=None, ge=1, le=5) + instruction_following: int | None = Field(default=None, ge=1, le=5) + person_object_count: int | None = Field(default=None, ge=1, le=5) + action_accuracy: int | None = Field(default=None, ge=1, le=5) + spatial_relation_accuracy: int | None = Field(default=None, ge=1, le=5) + classroom_usability: int | None = Field(default=None, ge=1, le=5) + visual_integrity: int | None = Field(default=None, ge=1, le=5) + cultural_age_appropriateness: int | None = Field(default=None, ge=1, le=5) + failure_labels: list[str] = Field(default_factory=list) + regeneration_required: bool | None = None + direct_courseware_use: bool | None = None + notes: str = Field(default="", max_length=3000) + + @model_validator(mode="after") + def _review_contract(self) -> TeacherReviewRecord: + if any(label not in _FAILURE_LABELS for label in self.failure_labels): + raise ValueError("unknown teacher failure label") + if len(set(self.failure_labels)) != len(self.failure_labels): + raise ValueError("teacher failure labels must be unique") + if self.review_state == "reviewed": + if any(getattr(self, field) is None for field in _RATING_FIELDS): + raise ValueError("reviewed records require all teacher ratings") + if self.regeneration_required is None or self.direct_courseware_use is None: + raise ValueError("reviewed records require use decisions") + return self + + +class BenchmarkRunState(_StrictModel): + schema_: Literal["hanclassstudio.teaching_image_benchmark_run.v1"] = Field( + default="hanclassstudio.teaching_image_benchmark_run.v1", alias="schema" + ) + run_id: str = Field(pattern=r"^[a-z0-9-]{8,80}$") + benchmark_id: str + benchmark_version: str + spec_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + output_dir: str + project_dir: str + selected_case_ids: list[str] + selected_case_seeds: dict[str, list[int]] + execution_identity: BenchmarkObservedIdentity | None = None + status: Literal["running", "paused", "completed", "blocked"] = "running" + results: dict[str, BenchmarkCaseResult] = Field(default_factory=dict) + started_at: str + updated_at: str + block: BenchmarkErrorRecord | None = None + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + + +def _sha256(value: Any) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _read_json(path: Path) -> Any: + try: + if path.stat().st_size > _MAX_JSON_BYTES: + raise BenchmarkError("json_too_large", f"JSON exceeds {_MAX_JSON_BYTES} bytes") + return json.loads(path.read_text(encoding="utf-8")) + except BenchmarkError: + raise + except (OSError, UnicodeDecodeError, ValueError) as exc: + raise BenchmarkError("json_invalid", f"Could not read JSON: {path}") from exc + + +def _write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + temporary.replace(path) + + +def _expected_fixed_identity() -> BenchmarkFixedIdentity: + runtime = load_runtime_manifest() + model = load_model_manifest() + workflow = load_workflow_pack() + return BenchmarkFixedIdentity( + model_package_id=model.package_id, + model_version=model.version, + model_sha256=model.source.sha256, + workflow_pack_id=workflow.pack_id, + workflow_version=workflow.version, + workflow_pack_sha256=WORKFLOW_PACK_SHA256, + runtime_id=runtime.runtime_id, + runtime_version=runtime.version, + runtime_source_commit=runtime.source_commit, + prompt_profile_id=workflow.prompt_profile.id, + ) + + +def load_benchmark_spec(path: Path) -> BenchmarkSpec: + try: + spec = BenchmarkSpec.model_validate(_read_json(path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("benchmark_schema_invalid", str(exc)) from exc + fixed = _expected_fixed_identity() + if spec.fixed_identity != fixed: + raise BenchmarkError( + "benchmark_identity_invalid", + "Benchmark fixed identity does not match the repository Model/Workflow/Runtime contracts", + ) + workflow = load_workflow_pack() + if spec.negative_prompt != workflow.prompt_profile.negative: + raise BenchmarkError( + "benchmark_prompt_invalid", "Benchmark negative prompt differs from the fixed Workflow Pack" + ) + return spec + + +def _asset_id(case_id: str, seed: int) -> str: + return f"bmk-{case_id}-{seed}" + + +def _case_key(case_id: str, seed: int) -> str: + return f"{case_id}@{seed}" + + +def _request_for(case: BenchmarkCase, seed: int) -> TeachingImageRequest: + return TeachingImageRequest( + asset_id=_asset_id(case.case_id, seed), + purpose=case.request.purpose, + subject=case.request.subject, + action=case.request.action, + environment=case.request.environment, + aspect_ratio=case.request.aspect_ratio, + seed=seed, + source_trace=[ + "benchmark:phase2c1-teaching-image-quality", + f"case:{case.case_id}", + f"seed:{seed}", + ], + ) + + +def _capture_identity() -> BenchmarkObservedIdentity: + try: + capability = generation_capability_snapshot(deep=True) + if not capability.generation_ready: + error = capability.technical_error or { + "code": "generation_not_ready", + "message": "Runtime, model, and Workflow are not jointly ready", + } + raise BenchmarkBlockedError(error["code"], error["message"]) + runtime = runtime_snapshot(recover=False) + model = validate_model_installation(deep=False) + workflow = load_workflow_pack() + fixed = _expected_fixed_identity() + if ( + runtime.actual_port is None + or runtime.process_identity is None + or runtime.version != fixed.runtime_version + or runtime.source_commit != fixed.runtime_source_commit + or model.package_id != fixed.model_package_id + or model.version != fixed.model_version + or model.model_sha256 != fixed.model_sha256 + or workflow.pack_id != fixed.workflow_pack_id + or workflow.version != fixed.workflow_version + ): + raise BenchmarkBlockedError( + "benchmark_identity_invalid", + "Live Runtime, model, or Workflow identity does not match the fixed benchmark", + ) + return BenchmarkObservedIdentity( + **fixed.model_dump(), + runtime_installation_identity=runtime_installation_identity(), + runtime_process_identity=runtime.process_identity, + runtime_port=runtime.actual_port, + model_installation_identity=model_installation_identity(model), + checked_at=_iso(), + ) + except BenchmarkError: + raise + except (ComfyUIModelError, ComfyUIRuntimeError, OSError, ValueError) as exc: + code = getattr(exc, "code", "generation_not_ready") + message = getattr(exc, "message", str(exc)) + raise BenchmarkBlockedError(code, message) from exc + + +def _same_execution_identity( + expected: BenchmarkObservedIdentity, observed: BenchmarkObservedIdentity +) -> bool: + return expected.model_dump(exclude={"checked_at"}) == observed.model_dump( + exclude={"checked_at"} + ) + + +def _technical_check( + project_dir: Path, + case: BenchmarkCase, + seed: int, + artifact: VerifiedImageArtifact, +) -> BenchmarkTechnicalCheck: + workflow = load_workflow_pack() + image_path = project_dir / artifact.path + provenance_path = project_dir / artifact.provenance_ref + if not image_path.is_file() or not provenance_path.is_file(): + raise BenchmarkError("artifact_missing", "Generated image or provenance file is missing") + payload = image_path.read_bytes() + verified = verify_png( + payload, + expected_width=workflow.dimensions[case.request.aspect_ratio][0], + expected_height=workflow.dimensions[case.request.aspect_ratio][1], + maximum_bytes=workflow.output.maximum_bytes, + ) + if verified.sha256 != artifact.sha256 or verified.size_bytes != artifact.size_bytes: + raise BenchmarkError("artifact_hash_mismatch", "Image hash or size differs from artifact") + provenance_bytes = provenance_path.read_bytes() + provenance = TeachingImageProvenance.model_validate_json(provenance_bytes) + provenance_sha = hashlib.sha256(provenance_bytes).hexdigest() + if provenance_sha != artifact.provenance_sha256: + raise BenchmarkError("provenance_hash_mismatch", "Provenance hash differs from artifact") + request = _request_for(case, seed) + request_sha = _sha256(request.model_dump(mode="json", by_alias=True)) + if provenance.request_sha256 != request_sha: + raise BenchmarkError("request_provenance_mismatch", "Provenance does not identify this case request") + if provenance.negative_prompt != case.negative_prompt: + raise BenchmarkError("negative_prompt_mismatch", "Provenance negative prompt differs from benchmark") + manifest_path = project_dir / "assets/data/asset_manifest.json" + manifest = AssetManifest.model_validate_json(manifest_path.read_bytes()) + matches = [asset for asset in manifest.images if asset.id == request.asset_id] + if len(matches) != 1 or matches[0].review_state != "pending_review": + raise BenchmarkError("manifest_registration_invalid", "Asset Manifest entry is missing or not pending_review") + registered = matches[0].verified_image_artifact + if registered is None or registered.artifact_id != artifact.artifact_id: + raise BenchmarkError("manifest_registration_invalid", "Manifest artifact does not match execution artifact") + return BenchmarkTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=verified.width, + height=verified.height, + image_size_bytes=verified.size_bytes, + image_sha256=verified.sha256, + provenance_sha256=provenance_sha, + manifest_asset_id=request.asset_id, + request_sha256=provenance.request_sha256, + execution_plan_sha256=provenance.execution_plan_sha256, + checks=[ + "png_signature_crc_dimensions", + "image_sha256", + "provenance_sha256_and_identity", + "asset_manifest_single_pending_review_entry", + "fixed_negative_prompt", + ], + ) + + +def _result_is_reusable( + state: BenchmarkRunState, + result: BenchmarkCaseResult, + case: BenchmarkCase, + project_dir: Path, +) -> bool: + if result.status != "succeeded" or result.artifact is None: + return False + try: + _technical_check(project_dir, case, result.seed, result.artifact) + except (BenchmarkError, OSError, ValueError): + return False + return True + + +def _record_error(code: str, message: str, attempt: int, recoverable: bool = True) -> BenchmarkErrorRecord: + return BenchmarkErrorRecord( + code=code, + message=message, + attempt=attempt, + recoverable=recoverable, + occurred_at=_iso(), + ) + + +def run_benchmark( + spec_path: Path, + output_dir: Path, + *, + case_ids: list[str] | None = None, + case_limit: int | None = None, + max_attempts: int = 2, +) -> BenchmarkRunState: + """Run or resume a benchmark, saving state after every attempt.""" + if max_attempts < 1 or max_attempts > 3: + raise BenchmarkError("invalid_attempts", "max_attempts must be between 1 and 3") + spec = load_benchmark_spec(spec_path) + spec_sha = _sha256(spec.model_dump(mode="json", by_alias=True)) + output_dir = output_dir.resolve() + project_dir = output_dir / "project" + state_path = output_dir / "run-state.json" + case_by_id = {case.case_id: case for case in spec.cases} + selected = case_ids or [case.case_id for case in spec.cases] + if case_limit is not None: + selected = selected[:case_limit] + unknown = sorted(set(selected) - set(case_by_id)) + if unknown: + raise BenchmarkError("unknown_case", f"Unknown benchmark cases: {', '.join(unknown)}") + selected = list(dict.fromkeys(selected)) + selected_seeds = {case_id: case_by_id[case_id].seeds for case_id in selected} + + if state_path.exists(): + try: + state = BenchmarkRunState.model_validate(_read_json(state_path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("run_state_invalid", str(exc)) from exc + if state.spec_sha256 != spec_sha or state.selected_case_ids != selected: + raise BenchmarkError("run_state_mismatch", "Existing run state belongs to a different benchmark selection") + if state.selected_case_seeds != selected_seeds: + raise BenchmarkError("run_state_mismatch", "Existing run state has different case seeds") + if state.status == "blocked": + raise BenchmarkBlockedError( + state.block.code if state.block else "benchmark_blocked", + state.block.message if state.block else "Benchmark is blocked; start a new run after restoring identity", + ) + if state.execution_identity is None: + state.execution_identity = _capture_identity() + else: + observed = _capture_identity() + if not _same_execution_identity(state.execution_identity, observed): + state.status = "blocked" + state.block = _record_error( + "benchmark_identity_changed", + "Runtime process, installation, model installation, or fixed package identity changed; start a new run", + max((result.attempts for result in state.results.values()), default=0) + 1, + recoverable=False, + ) + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise BenchmarkBlockedError(state.block.code, state.block.message) + was_completed = state.status == "completed" + state.status = "running" + else: + was_completed = False + observed = _capture_identity() + now = _iso() + state = BenchmarkRunState( + run_id=f"phase2c1-{int(time.time())}-{uuid.uuid4().hex[:8]}", + benchmark_id=spec.benchmark_id, + benchmark_version=spec.version, + spec_sha256=spec_sha, + output_dir=str(output_dir), + project_dir=str(project_dir), + selected_case_ids=selected, + selected_case_seeds=selected_seeds, + execution_identity=observed, + started_at=now, + updated_at=now, + ) + output_dir.mkdir(parents=True, exist_ok=True) + project_dir.mkdir(parents=True, exist_ok=True) + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + + try: + for case_id in selected: + case = case_by_id[case_id] + for seed in case.seeds: + key = _case_key(case_id, seed) + result = state.results.get(key) or BenchmarkCaseResult(case_id=case_id, seed=seed) + state.results[key] = result + if _result_is_reusable(state, result, case, project_dir): + continue + if was_completed and result.status in {"failed", "invalidated"}: + # A completed run is a terminal snapshot. A later invocation + # is an explicit retry of its failed/invalidated cases, while + # a paused run keeps its cumulative attempt count for resume. + result.attempts = 0 + if result.status == "succeeded": + result.status = "invalidated" + result.artifact = None + result.technical_checks = None + result.status = "running" + result.started_at = result.started_at or _iso() + result.error = None + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + for attempt in range(result.attempts + 1, max_attempts + 1): + result.attempts = attempt + try: + observed = _capture_identity() + if state.execution_identity is None or not _same_execution_identity( + state.execution_identity, observed + ): + raise BenchmarkBlockedError( + "benchmark_identity_changed", + "Runtime or installation identity changed during the benchmark", + ) + artifact = generate_teaching_image(project_dir, _request_for(case, seed)) + checks = _technical_check(project_dir, case, seed, artifact) + result.status = "succeeded" + result.completed_at = _iso() + result.artifact = artifact + result.technical_checks = checks + result.error = None + break + except BenchmarkBlockedError as exc: + result.status = "invalidated" + result.error = _record_error(exc.code, exc.message, attempt, recoverable=False) + state.status = "blocked" + state.block = result.error + raise + except (BenchmarkError, TeachingImageError, ComfyUIModelError, ComfyUIRuntimeError, OSError, ValueError) as exc: + code = getattr(exc, "code", "benchmark_execution_failed") + message = getattr(exc, "message", str(exc)) + result.error = _record_error(code, message, attempt) + result.status = "failed" + result.artifact = None + result.technical_checks = None + if attempt < max_attempts: + continue + finally: + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + except KeyboardInterrupt: + state.status = "paused" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + except BenchmarkBlockedError: + state.status = "blocked" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + raise + state.status = "completed" + state.updated_at = _iso() + _write_json(state_path, state.model_dump(mode="json", by_alias=True)) + return state + + +def _load_state(output_dir: Path) -> BenchmarkRunState: + path = output_dir / "run-state.json" + try: + return BenchmarkRunState.model_validate(_read_json(path)) + except (ValueError, TypeError) as exc: + raise BenchmarkError("run_state_invalid", str(exc)) from exc + + +def _review_record_for(key: str, result: BenchmarkCaseResult) -> TeacherReviewRecord: + if result.artifact is None: + raise BenchmarkError("review_artifact_missing", f"No successful artifact for {key}") + return TeacherReviewRecord( + case_key=key, + case_id=result.case_id, + seed=result.seed, + artifact_id=result.artifact.artifact_id, + ) + + +def write_review_package(spec_path: Path, output_dir: Path) -> Path: + spec = load_benchmark_spec(spec_path) + state = _load_state(output_dir) + package_dir = output_dir / "review-package" + images_dir = package_dir / "images" + images_dir.mkdir(parents=True, exist_ok=True) + case_by_id = {case.case_id: case for case in spec.cases} + records: list[TeacherReviewRecord] = [] + cards: list[dict[str, Any]] = [] + project_dir = Path(state.project_dir) + for key, result in state.results.items(): + if result.status != "succeeded" or result.artifact is None: + continue + record = _review_record_for(key, result) + records.append(record) + case = case_by_id[result.case_id] + source = project_dir / result.artifact.path + target = images_dir / f"{key.replace('@', '-')}.png" + if not source.is_file(): + raise BenchmarkError("review_image_missing", f"Review image is missing: {source}") + shutil.copyfile(source, target) + cards.append( + { + "key": key, + "case": case.model_dump(mode="json", by_alias=True), + "seed": result.seed, + "artifact_id": result.artifact.artifact_id, + "image": f"images/{target.name}", + } + ) + _write_json( + package_dir / "teacher-reviews.pending.json", + { + "schema": REVIEW_SCHEMA, + "benchmark_id": spec.benchmark_id, + "benchmark_version": spec.version, + "run_id": state.run_id, + "records": [record.model_dump(mode="json", by_alias=True) for record in records], + }, + ) + readme = ( + "# Phase 2C.1 教师图片评审包\n\n" + "本包只包含真实生成图片、案例合同和空白评审记录。所有评分字段初始为空," + "`pending_review` 不是通过结论。教师完成评分后,在界面点击 Download reviews.json," + "再使用 `benchmark_phase2c1.py report --review-file ` 导入。\n\n" + "评审维度:教学目标相关性、指令遵循、人物/物体数量、动作、空间关系、课堂可用性、" + "视觉完整性、文化与年龄适切性、是否需要重新生成、是否可直接用于课件。\n" + ) + (package_dir / "README.md").write_text(readme, encoding="utf-8") + (package_dir / "index.html").write_text(_review_html(cards), encoding="utf-8") + return package_dir + + +def _review_html(cards: list[dict[str, Any]]) -> str: + encoded = json.dumps(cards, ensure_ascii=False).replace("<", "\\u003c") + labels = sorted(_FAILURE_LABELS) + score_fields = [ + ("goal_relevance", "教学目标相关性"), + ("instruction_following", "指令遵循"), + ("person_object_count", "人物和物体数量"), + ("action_accuracy", "动作准确性"), + ("spatial_relation_accuracy", "空间关系准确性"), + ("classroom_usability", "课堂可用性"), + ("visual_integrity", "视觉完整性"), + ("cultural_age_appropriateness", "文化与年龄适切性"), + ] + score_markup = "".join( + f'" + for field, label in score_fields + ) + failure_markup = "".join( + f'' + for label in labels + ) + return f""" + +Phase 2C.1 Teacher Image Review + +

Phase 2C.1 教师图片质量评审

+

所有评分初始为空。自动技术检查不等于教师结论;只有教师明确点击“标记已评审”后才会产生 reviewed 记录。

+
+""" + + +def _load_reviews(path: Path) -> list[TeacherReviewRecord]: + raw = _read_json(path) + if not isinstance(raw, dict) or raw.get("schema") != REVIEW_SCHEMA or not isinstance(raw.get("records"), list): + raise BenchmarkError("review_file_invalid", "Teacher review file has an unknown schema") + try: + return [TeacherReviewRecord.model_validate(item) for item in raw["records"]] + except (ValueError, TypeError) as exc: + raise BenchmarkError("review_file_invalid", str(exc)) from exc + + +def aggregate_benchmark( + spec_path: Path, + output_dir: Path, + *, + review_path: Path | None = None, +) -> dict[str, Any]: + spec = load_benchmark_spec(spec_path) + state = _load_state(output_dir) + case_by_id = {case.case_id: case for case in spec.cases} + category_counts: dict[str, dict[str, Any]] = defaultdict( + lambda: {"total": 0, "succeeded": 0, "failed": 0, "pending": 0, "technical_success_rate": None} + ) + technical_failures: Counter[str] = Counter() + for result in state.results.values(): + category = case_by_id[result.case_id].category + counts = category_counts[category] + counts["total"] += 1 + if result.status == "succeeded": + counts["succeeded"] += 1 + elif result.status == "failed": + counts["failed"] += 1 + if result.error: + technical_failures[result.error.code] += 1 + else: + counts["pending"] += 1 + for counts in category_counts.values(): + if counts["total"]: + counts["technical_success_rate"] = round(counts["succeeded"] / counts["total"], 4) + reviews: list[TeacherReviewRecord] = [] + if review_path is not None: + reviews = _load_reviews(review_path) + expected = { + key: result + for key, result in state.results.items() + if result.status == "succeeded" and result.artifact is not None + } + for review in reviews: + if review.case_key not in expected: + raise BenchmarkError("review_target_invalid", f"Review targets unknown or failed case: {review.case_key}") + if expected[review.case_key].artifact.artifact_id != review.artifact_id: + raise BenchmarkError("review_artifact_mismatch", f"Review artifact changed for {review.case_key}") + report = { + "schema": "hanclassstudio.teaching_image_benchmark_report.v1", + "benchmark_id": spec.benchmark_id, + "benchmark_version": spec.version, + "spec_sha256": _sha256(spec.model_dump(mode="json", by_alias=True)), + "run_id": state.run_id, + "run_status": state.status, + "generated_at": _iso(), + "selected_cases": len(state.results), + "succeeded": sum(result.status == "succeeded" for result in state.results.values()), + "failed": sum(result.status == "failed" for result in state.results.values()), + "pending_or_invalidated": sum(result.status not in {"succeeded", "failed"} for result in state.results.values()), + "technical_success_rate": round( + sum(result.status == "succeeded" for result in state.results.values()) / len(state.results), 4 + ) + if state.results + else None, + "category_results": dict(sorted(category_counts.items())), + "technical_failure_frequency": dict(sorted(technical_failures.items())), + "teacher_review": { + "conclusion": None if not reviews else "teacher_data_imported", + "records_imported": len(reviews), + "records_pending_without_teacher": max( + 0, + sum(result.status == "succeeded" for result in state.results.values()) + - sum(review.review_state == "reviewed" for review in reviews), + ), + "automatic_teacher_scores": False, + "failure_label_frequency": dict( + sorted(Counter(label for review in reviews for label in review.failure_labels).items()) + ), + }, + "review_package": str((output_dir / "review-package").resolve()), + "limitations": [ + "Technical success is not teaching quality success.", + "Teacher ratings and failure labels remain empty until a teacher submits the review package.", + "This benchmark does not change the fixed model, Workflow Pack, or production prompt profile.", + ], + } + _write_json(output_dir / "benchmark-report.json", report) + return report + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Phase 2C.1 controlled teaching-image benchmark") + sub = parser.add_subparsers(dest="command", required=True) + run = sub.add_parser("run", help="run or resume a benchmark") + run.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + run.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + run.add_argument("--case-id", action="append", dest="case_ids") + run.add_argument("--case-limit", type=int) + run.add_argument("--max-attempts", type=int, default=2) + package = sub.add_parser("review-package", help="build portable teacher review package") + package.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + package.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + report = sub.add_parser("report", help="aggregate technical results and optional teacher reviews") + report.add_argument("--spec", type=Path, default=Path("benchmarks/phase2c1/cases.v1.json")) + report.add_argument("--output-dir", type=Path, default=Path("runtime/phase2c1-benchmark")) + report.add_argument("--review-file", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + if args.command == "run": + state = run_benchmark( + args.spec, + args.output_dir, + case_ids=args.case_ids, + case_limit=args.case_limit, + max_attempts=args.max_attempts, + ) + print(json.dumps(state.model_dump(mode="json", by_alias=True), ensure_ascii=False, indent=2)) + elif args.command == "review-package": + print(write_review_package(args.spec, args.output_dir)) + elif args.command == "report": + print(json.dumps(aggregate_benchmark(args.spec, args.output_dir, review_path=args.review_file), ensure_ascii=False, indent=2)) + return 0 + except BenchmarkBlockedError as exc: + print(f"BLOCKED [{exc.code}]: {exc.message}", file=sys.stderr) + return 2 + except BenchmarkError as exc: + print(f"ERROR [{exc.code}]: {exc.message}", file=sys.stderr) + return 1 + except KeyboardInterrupt: + print("PAUSED: interrupt received; run state was preserved", file=sys.stderr) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/apps/api/tests/test_comfyui_teaching_image.py b/apps/api/tests/test_comfyui_teaching_image.py index 00c195a..2a27c4b 100644 --- a/apps/api/tests/test_comfyui_teaching_image.py +++ b/apps/api/tests/test_comfyui_teaching_image.py @@ -256,6 +256,31 @@ def fake_json(_port, method, path, **kwargs): assert len(viewed) == 1 +def test_executor_cancels_current_job_when_batch_is_interrupted(monkeypatch) -> None: + _ready_runtime(monkeypatch) + plan = images.compile_teaching_image_request( + _request(), model_record=_record(), workflow=load_workflow_pack() + ) + submitted: dict[str, object] = {} + cancelled: list[str] = [] + + def fake_json(_port, method, path, **kwargs): + if method == "POST" and path == "/prompt": + submitted.update(kwargs["payload"]) + return {"prompt_id": submitted["prompt_id"], "node_errors": {}} + raise KeyboardInterrupt + + monkeypatch.setattr(images, "_http_json", fake_json) + monkeypatch.setattr( + images, + "_cancel_job_if_still_owned", + lambda _plan, prompt_id: cancelled.append(prompt_id), + ) + with pytest.raises(KeyboardInterrupt): + images._execute_fixed_plan(plan, 8188, 16 * 1024**2) + assert cancelled == [submitted["prompt_id"]] + + @pytest.mark.parametrize( "mismatch", ["client", "graph", "prior_output", "path_traversal"], diff --git a/apps/api/tests/test_teaching_image_benchmark.py b/apps/api/tests/test_teaching_image_benchmark.py new file mode 100644 index 0000000..b1085e4 --- /dev/null +++ b/apps/api/tests/test_teaching_image_benchmark.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from hcs_api import teaching_image_benchmark as benchmark +from hcs_api.comfyui_teaching_image import TeachingImageError +from hcs_api.models import TeachingImageProvenance, VerifiedImageArtifact + +ROOT = Path(__file__).resolve().parents[3] +SPEC_PATH = ROOT / "benchmarks/phase2c1/cases.v1.json" + + +def _identity(checked_at: str = "2026-07-27T00:00:00+00:00") -> benchmark.BenchmarkObservedIdentity: + fixed = benchmark._expected_fixed_identity() + return benchmark.BenchmarkObservedIdentity( + **fixed.model_dump(), + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_installation_identity="c" * 64, + checked_at=checked_at, + ) + + +def _artifact(asset_id: str, seed: int) -> VerifiedImageArtifact: + provenance = TeachingImageProvenance( + runtime_version="0.28.0", + runtime_source_commit="700821e1364eaab0e8f21c538a2131719fec57bf", + runtime_installation_identity="a" * 64, + runtime_process_identity="b" * 64, + runtime_port=8188, + model_package_id="hcs.sd15-teaching-illustration-fp16", + model_version="1.5-fp16-emaonly", + model_manifest_sha256="1" * 64, + model_sha256="e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + model_installation_identity="c" * 64, + workflow_pack_id="hcs.teaching-illustration-sd15-core", + workflow_version="1.0.0", + workflow_pack_sha256="e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + prompt_profile_id="soft-flat-educational-v1", + positive_prompt="fixed positive prompt", + negative_prompt="text, letters, words", + seed=seed, + steps=20, + cfg=7.0, + sampler_name="euler", + scheduler="normal", + denoise=1.0, + output_prefix="hcs_" + "4" * 20 + "_" + "5" * 12, + prompt_id="11111111-1111-4111-8111-111111111111", + source_trace=[f"case:{asset_id}"], + ) + return VerifiedImageArtifact( + artifact_id="img-" + "6" * 24, + asset_id=asset_id, + path=f"assets/images/{asset_id}.png", + width=512, + height=384, + size_bytes=100, + sha256="7" * 64, + provenance_ref=f"assets/data/image-provenance/{asset_id}.json", + provenance_sha256="8" * 64, + provenance=provenance, + ) + + +def _checks(artifact: VerifiedImageArtifact) -> benchmark.BenchmarkTechnicalCheck: + return benchmark.BenchmarkTechnicalCheck( + image_path=artifact.path, + provenance_path=artifact.provenance_ref, + width=artifact.width, + height=artifact.height, + image_size_bytes=artifact.size_bytes, + image_sha256=artifact.sha256, + provenance_sha256=artifact.provenance_sha256, + manifest_asset_id=artifact.asset_id, + request_sha256="2" * 64, + execution_plan_sha256="3" * 64, + checks=["fixture"], + ) + + +def test_benchmark_spec_has_fixed_identity_and_all_required_categories() -> None: + spec = benchmark.load_benchmark_spec(SPEC_PATH) + assert len(spec.cases) == 25 + assert {case.category for case in spec.cases} == benchmark._REQUIRED_CATEGORIES + assert all(case.prompt.startswith("subject: ") for case in spec.cases) + assert all(case.negative_prompt == spec.negative_prompt for case in spec.cases) + + +def test_execution_identity_ignores_check_timestamp_but_not_runtime_process() -> None: + assert benchmark._same_execution_identity(_identity(), _identity("later")) + changed = _identity() + changed.runtime_process_identity = "d" * 64 + assert not benchmark._same_execution_identity(_identity(), changed) + + +def test_run_continues_after_one_case_failure_and_resumes_idempotently( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + spec = benchmark.load_benchmark_spec(SPEC_PATH) + selected = [spec.cases[0].case_id, spec.cases[1].case_id] + observed = _identity() + monkeypatch.setattr(benchmark, "_capture_identity", lambda: observed) + calls: list[str] = [] + fail_apple = True + + def fake_generate(project_dir: Path, request): + nonlocal fail_apple + calls.append(request.asset_id) + if request.asset_id.startswith("bmk-obj_apple") and fail_apple: + raise TeachingImageError("generation_failed", "fixture failure") + return _artifact(request.asset_id, request.seed) + + monkeypatch.setattr(benchmark, "generate_teaching_image", fake_generate) + monkeypatch.setattr(benchmark, "_technical_check", lambda *_args: _checks(_args[3])) + output = tmp_path / "run" + state = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert state.status == "completed" + assert state.results[f"{selected[0]}@260101"].status == "failed" + assert state.results[f"{selected[1]}@260102"].status == "succeeded" + assert len(calls) == 2 + + fail_apple = False + resumed = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert resumed.status == "completed" + assert resumed.results[f"{selected[0]}@260101"].status == "succeeded" + assert len(calls) == 3 + + idempotent = benchmark.run_benchmark(SPEC_PATH, output, case_ids=selected, max_attempts=1) + assert idempotent.status == "completed" + assert len(calls) == 3 + + +def test_identity_change_blocks_unfinished_cases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + observed = _identity() + changed = _identity() + changed.runtime_process_identity = "d" * 64 + captures = iter([observed, changed]) + monkeypatch.setattr(benchmark, "_capture_identity", lambda: next(captures)) + with pytest.raises(benchmark.BenchmarkBlockedError, match="identity"): + benchmark.run_benchmark( + SPEC_PATH, + tmp_path / "run", + case_ids=["obj_apple_01"], + max_attempts=1, + ) + state = benchmark.BenchmarkRunState.model_validate( + json.loads((tmp_path / "run/run-state.json").read_text(encoding="utf-8")) + ) + assert state.status == "blocked" + assert state.block and state.block.code == "benchmark_identity_changed" + + +def test_teacher_review_defaults_are_empty_and_reviewed_requires_all_fields() -> None: + pending = benchmark.TeacherReviewRecord( + case_key="obj_apple_01@260101", + case_id="obj_apple_01", + seed=260101, + artifact_id="img-" + "6" * 24, + ) + assert pending.review_state == "pending_review" + assert pending.goal_relevance is None + with pytest.raises(ValueError, match="all teacher ratings"): + reviewed_payload = pending.model_dump() + reviewed_payload["review_state"] = "reviewed" + benchmark.TeacherReviewRecord.model_validate(reviewed_payload) + + +def test_report_does_not_infer_teacher_conclusions(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + observed = _identity() + monkeypatch.setattr(benchmark, "_capture_identity", lambda: observed) + artifact = _artifact("bmk-obj_apple_01-260101", 260101) + monkeypatch.setattr(benchmark, "generate_teaching_image", lambda *_args: artifact) + monkeypatch.setattr(benchmark, "_technical_check", lambda *_args: _checks(artifact)) + state = benchmark.run_benchmark( + SPEC_PATH, + tmp_path / "run", + case_ids=["obj_apple_01"], + max_attempts=1, + ) + assert state.status == "completed" + report = benchmark.aggregate_benchmark(SPEC_PATH, tmp_path / "run") + assert report["technical_success_rate"] == 1.0 + assert report["teacher_review"]["conclusion"] is None + assert report["teacher_review"]["records_pending_without_teacher"] == 1 + assert report["teacher_review"]["failure_label_frequency"] == {} diff --git a/benchmarks/phase2c1/cases.v1.json b/benchmarks/phase2c1/cases.v1.json new file mode 100644 index 0000000..7b233ff --- /dev/null +++ b/benchmarks/phase2c1/cases.v1.json @@ -0,0 +1,1032 @@ +{ + "schema": "hanclassstudio.teaching_image_benchmark.v1", + "benchmark_id": "phase2c1-teaching-image-quality", + "version": "1.0.0", + "fixed_identity": { + "model_package_id": "hcs.sd15-teaching-illustration-fp16", + "model_version": "1.5-fp16-emaonly", + "model_sha256": "e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916", + "workflow_pack_id": "hcs.teaching-illustration-sd15-core", + "workflow_version": "1.0.0", + "workflow_pack_sha256": "e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e", + "runtime_id": "comfyui", + "runtime_version": "0.28.0", + "runtime_source_commit": "700821e1364eaab0e8f21c538a2131719fec57bf", + "prompt_profile_id": "soft-flat-educational-v1" + }, + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity", + "cases": [ + { + "case_id": "obj_apple_01", + "category": "single_object", + "title": "苹果", + "teaching_goal": "Learner can recognize 苹果 as a single everyday object.", + "prompt": "subject: one red apple; action: resting on a small table; environment: a plain bright teaching surface", + "request": { + "purpose": "vocabulary_image", + "subject": "one red apple", + "action": "resting on a small table", + "environment": "a plain bright teaching surface", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260101 + ], + "expected": { + "people": 0, + "objects": [ + "one red apple", + "small table" + ], + "actions": [], + "relations": [], + "scene": "plain teaching surface" + }, + "must_satisfy": [ + "one clearly recognizable red apple is the focal object", + "no extra people are present" + ], + "severe_failures": [ + "missing apple", + "multiple apples when singularity is unclear", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "obj_book_01", + "category": "single_object", + "title": "书", + "teaching_goal": "Learner can recognize 书 as a single classroom object.", + "prompt": "subject: one blue book; action: open on a desk; environment: a simple classroom table", + "request": { + "purpose": "vocabulary_image", + "subject": "one blue book", + "action": "open on a desk", + "environment": "a simple classroom table", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260102 + ], + "expected": { + "people": 0, + "objects": [ + "one blue book", + "desk" + ], + "actions": [ + "open" + ], + "relations": [ + "book on desk" + ], + "scene": "simple classroom table" + }, + "must_satisfy": [ + "book is the clear focal object", + "book is visibly open on a desk" + ], + "severe_failures": [ + "book absent", + "book replaced by a phone or laptop", + "unreadable fake writing dominating the image" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "obj_umbrella_01", + "category": "single_object", + "title": "雨伞", + "teaching_goal": "Learner can recognize 雨伞 as an everyday object.", + "prompt": "subject: one yellow umbrella; action: standing closed beside a doorway; environment: a clean home entryway", + "request": { + "purpose": "vocabulary_image", + "subject": "one yellow umbrella", + "action": "standing closed beside a doorway", + "environment": "a clean home entryway", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260103 + ], + "expected": { + "people": 0, + "objects": [ + "one yellow umbrella", + "doorway" + ], + "actions": [], + "relations": [ + "umbrella beside doorway" + ], + "scene": "clean home entryway" + }, + "must_satisfy": [ + "umbrella silhouette is unmistakable", + "one umbrella is shown" + ], + "severe_failures": [ + "umbrella missing", + "several umbrellas with unclear count", + "umbrella becomes an unrelated object" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_wave_01", + "category": "person_action", + "title": "挥手", + "teaching_goal": "Learner can connect 挥手 with a person visibly waving.", + "prompt": "subject: one child; action: waving one raised hand hello; environment: a bright uncluttered classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one child", + "action": "waving one raised hand hello", + "environment": "a bright uncluttered classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260104 + ], + "expected": { + "people": 1, + "objects": [], + "actions": [ + "wave with one raised hand" + ], + "relations": [], + "scene": "bright uncluttered classroom" + }, + "must_satisfy": [ + "one child is visible", + "raised hand clearly communicates waving" + ], + "severe_failures": [ + "wrong action such as sitting or sleeping", + "extra people", + "severe hand anatomy defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_drink_01", + "category": "person_action", + "title": "喝水", + "teaching_goal": "Learner can connect 喝水 with a person drinking from a cup.", + "prompt": "subject: one adult student; action: drinking water from a cup; environment: a simple classroom desk", + "request": { + "purpose": "classroom_scene", + "subject": "one adult student", + "action": "drinking water from a cup", + "environment": "a simple classroom desk", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260105 + ], + "expected": { + "people": 1, + "objects": [ + "cup", + "desk" + ], + "actions": [ + "drink water" + ], + "relations": [ + "cup near mouth" + ], + "scene": "simple classroom desk" + }, + "must_satisfy": [ + "cup is visibly held near the mouth", + "one student is shown" + ], + "severe_failures": [ + "person not drinking", + "cup absent", + "dangerous or culturally inappropriate context" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "action_open_01", + "category": "person_action", + "title": "开门", + "teaching_goal": "Learner can connect 开门 with a person opening a door.", + "prompt": "subject: one person; action: opening a blue door with one hand; environment: a tidy apartment entrance", + "request": { + "purpose": "classroom_scene", + "subject": "one person", + "action": "opening a blue door with one hand", + "environment": "a tidy apartment entrance", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260106 + ], + "expected": { + "people": 1, + "objects": [ + "blue door" + ], + "actions": [ + "open door" + ], + "relations": [ + "hand touching door" + ], + "scene": "tidy apartment entrance" + }, + "must_satisfy": [ + "door and hand contact are clear", + "one person is shown" + ], + "severe_failures": [ + "door missing", + "person merely standing", + "extra limbs or severe anatomy defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_one_01", + "category": "person_count", + "title": "一个学生", + "teaching_goal": "Learner can distinguish one person from a group.", + "prompt": "subject: exactly one student; action: standing and smiling; environment: a plain classroom wall", + "request": { + "purpose": "classroom_scene", + "subject": "exactly one student", + "action": "standing and smiling", + "environment": "a plain classroom wall", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260107 + ], + "expected": { + "people": 1, + "objects": [], + "actions": [ + "stand", + "smile" + ], + "relations": [], + "scene": "plain classroom wall" + }, + "must_satisfy": [ + "exactly one human figure is visible", + "figure is standing" + ], + "severe_failures": [ + "wrong_count", + "crowd or partial extra person", + "face or body not visually complete" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_two_01", + "category": "person_count", + "title": "两个朋友", + "teaching_goal": "Learner can identify two people in a simple scene.", + "prompt": "subject: exactly two friends; action: standing side by side and smiling; environment: a simple park path", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two friends", + "action": "standing side by side and smiling", + "environment": "a simple park path", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260108 + ], + "expected": { + "people": 2, + "objects": [], + "actions": [ + "stand side by side", + "smile" + ], + "relations": [ + "two people side by side" + ], + "scene": "simple park path" + }, + "must_satisfy": [ + "exactly two complete people are visible", + "people stand side by side" + ], + "severe_failures": [ + "wrong_count", + "one person or crowd", + "people merge into an unreadable figure" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "count_three_01", + "category": "person_count", + "title": "三个学生", + "teaching_goal": "Learner can identify three people in a classroom.", + "prompt": "subject: exactly three students; action: sitting together at one table; environment: a bright language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "exactly three students", + "action": "sitting together at one table", + "environment": "a bright language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260109 + ], + "expected": { + "people": 3, + "objects": [ + "one table" + ], + "actions": [ + "sit together" + ], + "relations": [ + "three people around one table" + ], + "scene": "bright language classroom" + }, + "must_satisfy": [ + "three complete people are discernible", + "one shared table is visible" + ], + "severe_failures": [ + "wrong_count", + "four or more people", + "table or people are missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_left_01", + "category": "spatial_relation", + "title": "左边", + "teaching_goal": "Learner can interpret 左边 using a clear left-right relation.", + "prompt": "subject: a red ball and a blue box; action: the red ball is to the left of the blue box; environment: a plain tabletop", + "request": { + "purpose": "vocabulary_image", + "subject": "a red ball and a blue box", + "action": "the red ball is to the left of the blue box", + "environment": "a plain tabletop", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260110 + ], + "expected": { + "people": 0, + "objects": [ + "red ball", + "blue box" + ], + "actions": [], + "relations": [ + "red ball left of blue box" + ], + "scene": "plain tabletop" + }, + "must_satisfy": [ + "both objects are separate and visible", + "red ball is clearly on the left" + ], + "severe_failures": [ + "wrong_spatial_relation", + "one object missing", + "objects overlap so relation is unclear" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_under_01", + "category": "spatial_relation", + "title": "下面", + "teaching_goal": "Learner can interpret 下面 using an above-below relation.", + "prompt": "subject: a cat and a chair; action: the cat is under the chair; environment: a clean living room", + "request": { + "purpose": "vocabulary_image", + "subject": "a cat and a chair", + "action": "the cat is under the chair", + "environment": "a clean living room", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260111 + ], + "expected": { + "people": 0, + "objects": [ + "cat", + "chair" + ], + "actions": [], + "relations": [ + "cat under chair" + ], + "scene": "clean living room" + }, + "must_satisfy": [ + "chair is above the cat", + "cat and chair are both recognizable" + ], + "severe_failures": [ + "wrong_spatial_relation", + "cat missing", + "chair missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "space_between_01", + "category": "spatial_relation", + "title": "中间", + "teaching_goal": "Learner can interpret 中间 in a three-object arrangement.", + "prompt": "subject: a small green plant between two books; action: the plant is in the middle of the books; environment: a neat desk", + "request": { + "purpose": "vocabulary_image", + "subject": "a small green plant between two books", + "action": "the plant is in the middle of the books", + "environment": "a neat desk", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260112 + ], + "expected": { + "people": 0, + "objects": [ + "small green plant", + "two books" + ], + "actions": [], + "relations": [ + "plant between two books" + ], + "scene": "neat desk" + }, + "must_satisfy": [ + "two books flank the plant", + "plant is visually central" + ], + "severe_failures": [ + "wrong_spatial_relation", + "fewer than two books", + "plant absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_read_01", + "category": "classroom_activity", + "title": "读书", + "teaching_goal": "Learner can recognize a classroom reading activity.", + "prompt": "subject: one teacher and one student; action: the student reads a book while the teacher listens; environment: a calm Chinese language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and one student", + "action": "the student reads a book while the teacher listens", + "environment": "a calm Chinese language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260113 + ], + "expected": { + "people": 2, + "objects": [ + "book", + "classroom desk" + ], + "actions": [ + "student reads", + "teacher listens" + ], + "relations": [ + "teacher and student at desk" + ], + "scene": "calm Chinese language classroom" + }, + "must_satisfy": [ + "book is visible", + "student and teacher roles are visually plausible", + "scene reads as a classroom" + ], + "severe_failures": [ + "wrong_scene", + "reading activity absent", + "crowd or no classroom cues" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_write_01", + "category": "classroom_activity", + "title": "写汉字", + "teaching_goal": "Learner can recognize a guided writing activity without relying on generated text.", + "prompt": "subject: one teacher and one student; action: the teacher points while the student writes in a notebook; environment: an uncluttered language classroom", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and one student", + "action": "the teacher points while the student writes in a notebook", + "environment": "an uncluttered language classroom", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260114 + ], + "expected": { + "people": 2, + "objects": [ + "notebook", + "pencil", + "desk" + ], + "actions": [ + "teacher points", + "student writes" + ], + "relations": [ + "student writes at desk" + ], + "scene": "uncluttered language classroom" + }, + "must_satisfy": [ + "writing posture and notebook are clear", + "no readable generated text is required", + "teacher-student arrangement is plausible" + ], + "severe_failures": [ + "text artifact", + "wrong_scene", + "writing action absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "class_pair_01", + "category": "classroom_activity", + "title": "两人对话", + "teaching_goal": "Learner can recognize a pair speaking activity.", + "prompt": "subject: exactly two language students; action: facing each other and practicing a short conversation; environment: a friendly classroom pair-work table", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two language students", + "action": "facing each other and practicing a short conversation", + "environment": "a friendly classroom pair-work table", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260115 + ], + "expected": { + "people": 2, + "objects": [ + "pair-work table" + ], + "actions": [ + "speak to each other" + ], + "relations": [ + "two students face each other" + ], + "scene": "friendly classroom pair-work table" + }, + "must_satisfy": [ + "exactly two students are visible", + "face-to-face orientation is clear", + "no speech text is needed" + ], + "severe_failures": [ + "wrong_count", + "text artifact", + "students face away from each other" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_greet_01", + "category": "daily_communication", + "title": "问候", + "teaching_goal": "Learner can recognize a polite everyday greeting.", + "prompt": "subject: one adult and one older adult; action: smiling and greeting each other respectfully at a doorway; environment: a welcoming home entrance", + "request": { + "purpose": "classroom_scene", + "subject": "one adult and one older adult", + "action": "smiling and greeting each other respectfully at a doorway", + "environment": "a welcoming home entrance", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260116 + ], + "expected": { + "people": 2, + "objects": [ + "doorway" + ], + "actions": [ + "greet respectfully", + "smile" + ], + "relations": [ + "two people face each other" + ], + "scene": "welcoming home entrance" + }, + "must_satisfy": [ + "greeting posture is readable", + "age relationship is respectful", + "no text is needed" + ], + "severe_failures": [ + "wrong_scene", + "people ignore each other", + "culturally inappropriate interaction" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_shop_01", + "category": "daily_communication", + "title": "买东西", + "teaching_goal": "Learner can recognize a simple shopping exchange.", + "prompt": "subject: one shopper and one shopkeeper; action: handing a small bag across a counter; environment: a clean neighborhood shop", + "request": { + "purpose": "classroom_scene", + "subject": "one shopper and one shopkeeper", + "action": "handing a small bag across a counter", + "environment": "a clean neighborhood shop", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260117 + ], + "expected": { + "people": 2, + "objects": [ + "small bag", + "shop counter" + ], + "actions": [ + "hand a bag" + ], + "relations": [ + "bag passes across counter" + ], + "scene": "clean neighborhood shop" + }, + "must_satisfy": [ + "counter and bag are visible", + "two roles are plausible", + "exchange is clear" + ], + "severe_failures": [ + "wrong_scene", + "bag absent", + "crowded unreadable shop" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "daily_bus_01", + "category": "daily_communication", + "title": "问路", + "teaching_goal": "Learner can recognize asking for directions in daily life.", + "prompt": "subject: one visitor and one local person; action: the visitor points at a simple map while asking for directions; environment: a quiet city street corner", + "request": { + "purpose": "classroom_scene", + "subject": "one visitor and one local person", + "action": "the visitor points at a simple map while asking for directions", + "environment": "a quiet city street corner", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260118 + ], + "expected": { + "people": 2, + "objects": [ + "simple map" + ], + "actions": [ + "point at map", + "ask for directions" + ], + "relations": [ + "two people look at map" + ], + "scene": "quiet city street corner" + }, + "must_satisfy": [ + "map is visible without readable text", + "visitor and local attend to each other" + ], + "severe_failures": [ + "text artifact", + "wrong_scene", + "map absent" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "emotion_happy_01", + "category": "emotion_expression", + "title": "高兴", + "teaching_goal": "Learner can associate 高兴 with a clearly happy expression.", + "prompt": "subject: one child; action: smiling broadly and holding a small gift; environment: a warm family room", + "request": { + "purpose": "vocabulary_image", + "subject": "one child", + "action": "smiling broadly and holding a small gift", + "environment": "a warm family room", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260119 + ], + "expected": { + "people": 1, + "objects": [ + "small gift" + ], + "actions": [ + "smile broadly", + "hold gift" + ], + "relations": [], + "scene": "warm family room" + }, + "must_satisfy": [ + "facial expression is visibly happy", + "gift is secondary and recognizable" + ], + "severe_failures": [ + "emotion unreadable", + "sad or frightened expression", + "severe face defect" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "emotion_tired_01", + "category": "emotion_expression", + "title": "累", + "teaching_goal": "Learner can associate 累 with a tired but safe everyday expression.", + "prompt": "subject: one office worker; action: sitting with tired shoulders and a cup of water; environment: a quiet desk after work", + "request": { + "purpose": "vocabulary_image", + "subject": "one office worker", + "action": "sitting with tired shoulders and a cup of water", + "environment": "a quiet desk after work", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260120 + ], + "expected": { + "people": 1, + "objects": [ + "cup", + "desk" + ], + "actions": [ + "sit tiredly" + ], + "relations": [ + "person at desk" + ], + "scene": "quiet desk after work" + }, + "must_satisfy": [ + "tired posture is clear", + "scene remains age-appropriate and safe" + ], + "severe_failures": [ + "emotion unreadable", + "person appears injured", + "visually confusing posture" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "culture_tea_01", + "category": "cultural_scene", + "title": "喝茶", + "teaching_goal": "Learner can recognize a respectful Chinese tea-sharing scene.", + "prompt": "subject: two adults; action: respectfully sharing tea at a small round table; environment: a calm Chinese tea room with simple ceramic cups", + "request": { + "purpose": "classroom_scene", + "subject": "two adults", + "action": "respectfully sharing tea at a small round table", + "environment": "a calm Chinese tea room with simple ceramic cups", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260121 + ], + "expected": { + "people": 2, + "objects": [ + "round table", + "ceramic cups", + "tea pot" + ], + "actions": [ + "share tea" + ], + "relations": [ + "two people around table" + ], + "scene": "calm Chinese tea room" + }, + "must_satisfy": [ + "tea objects are recognizable", + "interaction is respectful", + "no stereotypes or decorative text dominate" + ], + "severe_failures": [ + "culturally_inappropriate", + "wrong_scene", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "culture_festival_01", + "category": "cultural_scene", + "title": "春节", + "teaching_goal": "Learner can recognize a family celebration connected to Spring Festival.", + "prompt": "subject: three family members; action: sitting together for a respectful Spring Festival meal; environment: a warm Chinese family dining room with red decorations but no writing", + "request": { + "purpose": "classroom_scene", + "subject": "three family members", + "action": "sitting together for a respectful Spring Festival meal", + "environment": "a warm Chinese family dining room with red decorations but no writing", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260122 + ], + "expected": { + "people": 3, + "objects": [ + "dining table", + "shared meal", + "simple red decorations" + ], + "actions": [ + "sit together for meal" + ], + "relations": [ + "family around table" + ], + "scene": "warm Chinese family dining room" + }, + "must_satisfy": [ + "family meal is clear", + "decorations are simple and text-free", + "age and interaction are respectful" + ], + "severe_failures": [ + "culturally_inappropriate", + "wrong_count", + "text artifact or stereotyped costume" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "sequence_arrive_01", + "category": "event_sequence", + "title": "先后顺序", + "teaching_goal": "Learner can infer a simple before-and-after event from one visual scene.", + "prompt": "subject: one student arriving at school; action: holding a backpack at the school entrance just before entering; environment: a clear morning school doorway", + "request": { + "purpose": "classroom_scene", + "subject": "one student arriving at school", + "action": "holding a backpack at the school entrance just before entering", + "environment": "a clear morning school doorway", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260123 + ], + "expected": { + "people": 1, + "objects": [ + "backpack", + "school doorway" + ], + "actions": [ + "arrive before entering" + ], + "relations": [ + "student outside doorway" + ], + "scene": "clear morning school doorway" + }, + "must_satisfy": [ + "arrival-before-entry state is visually plausible", + "backpack and doorway are clear" + ], + "severe_failures": [ + "wrong_scene", + "student already inside with no doorway context", + "visually confusing composition" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "hard_count_action_space_01", + "category": "hard_combination", + "title": "数量+动作+方位", + "teaching_goal": "Learner can attempt a combined count, action, and spatial-relation prompt.", + "prompt": "subject: exactly two children and one red ball; action: one child stands left and passes the ball to the other child on the right; environment: a simple school playground", + "request": { + "purpose": "classroom_scene", + "subject": "exactly two children and one red ball", + "action": "one child stands left and passes the ball to the other child on the right", + "environment": "a simple school playground", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260124 + ], + "expected": { + "people": 2, + "objects": [ + "one red ball" + ], + "actions": [ + "pass ball" + ], + "relations": [ + "one child left of the other", + "ball between children" + ], + "scene": "simple school playground" + }, + "must_satisfy": [ + "exactly two children are visible", + "one ball is visible", + "left-right relation and pass action are at least interpretable" + ], + "severe_failures": [ + "wrong_count", + "wrong_action", + "wrong_spatial_relation", + "ball missing" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + }, + { + "case_id": "hard_role_culture_01", + "category": "hard_combination", + "title": "角色+文化+课堂", + "teaching_goal": "Learner can attempt a culturally respectful classroom role-play scene.", + "prompt": "subject: one teacher and two students; action: students politely greet the teacher before a Chinese lesson; environment: a calm classroom with a simple tea table and no written signs", + "request": { + "purpose": "classroom_scene", + "subject": "one teacher and two students", + "action": "students politely greet the teacher before a Chinese lesson", + "environment": "a calm classroom with a simple tea table and no written signs", + "aspect_ratio": "4:3" + }, + "seeds": [ + 260125 + ], + "expected": { + "people": 3, + "objects": [ + "simple tea table" + ], + "actions": [ + "students greet teacher politely" + ], + "relations": [ + "students face teacher" + ], + "scene": "calm Chinese lesson classroom" + }, + "must_satisfy": [ + "three people are discernible", + "teacher-student role relationship is plausible", + "scene is respectful and text-free" + ], + "severe_failures": [ + "wrong_count", + "culturally_inappropriate", + "wrong_scene", + "text artifact" + ], + "negative_prompt": "text, letters, words, captions, subtitles, logo, watermark, photorealistic, cluttered background, low quality, blurry, distorted hands, extra fingers, violence, weapons, nudity" + } + ] +} diff --git a/docs/phase2c1-teaching-image-benchmark-results.md b/docs/phase2c1-teaching-image-benchmark-results.md new file mode 100644 index 0000000..f8e3644 --- /dev/null +++ b/docs/phase2c1-teaching-image-benchmark-results.md @@ -0,0 +1,73 @@ +# Phase 2C.1 首轮运行结果 + +这份记录只报告可复现的运行和自动技术检查。它不把模型观察或自动规则称为教师教学结论。 + +## 固定身份 + +- Benchmark:`phase2c1-teaching-image-quality` `1.0.0` +- Model:`hcs.sd15-teaching-illustration-fp16` `1.5-fp16-emaonly` +- Model SHA-256:`e9476a13728cd75d8279f6ec8bad753a66a1957ca375a1464dc63b37db6e3916` +- Workflow:`hcs.teaching-illustration-sd15-core` `1.0.0` +- Workflow SHA-256:`e25c17976054ad0122c943a22631640afd50fa52af960e839ce29cd168c1751e` +- Runtime:ComfyUI `0.28.0`, source commit `700821e1364eaab0e8f21c538a2131719fec57bf` +- Prompt profile:`soft-flat-educational-v1` +- Platform:macOS arm64 opt-in Runtime lifecycle + +## Pilot + +`runtime/phase2c1-pilot-v2` 使用 3 个案例和固定 seed:`obj_apple_01`、`action_wave_01`、 +`class_read_01`。3/3 真实生成,3/3 通过 PNG、尺寸、图片哈希、provenance、固定身份和 Asset Manifest +技术检查;3 条评审记录均为 `pending_review`。 + +早先的一次 pilot 启动方式把 Runtime 放在已退出的一次性父进程中,ComfyUI 历史返回 +`execution_error` / `BrokenPipeError`。该结果 fail closed,没有创建可接受 artifact。改用持续 Runtime +所属会话后重新生成成功。运行时必须保持启动 Runtime 的父会话存活;这不是放宽 history 或 provenance +校验。 + +## 完整基准 + +Run state:`runtime/phase2c1-full/run-state.json`,run id +`phase2c1-1785095674-ee847dfe`。 + +| 类别 | 案例数 | 技术成功 | 技术失败 | +| --- | ---: | ---: | ---: | +| `single_object` | 3 | 3 | 0 | +| `person_action` | 3 | 3 | 0 | +| `person_count` | 3 | 3 | 0 | +| `spatial_relation` | 3 | 3 | 0 | +| `classroom_activity` | 3 | 3 | 0 | +| `daily_communication` | 3 | 3 | 0 | +| `emotion_expression` | 2 | 2 | 0 | +| `cultural_scene` | 2 | 2 | 0 | +| `event_sequence` | 1 | 1 | 0 | +| `hard_combination` | 2 | 2 | 0 | +| **合计** | **25** | **25** | **0** | + +每个案例使用 1 个固定 seed;总尝试数为 25。没有 pending、invalidated 或静默跳过案例。 +每张图片为 512×384 PNG,并在项目控制目录登记为 `pending_review` Asset Manifest 条目。 + +自动报告结论:`technical_success_rate = 1.0`、`technical_failure_frequency = {}`。 +自动检查覆盖 PNG signature/CRC/尺寸/大小、图片 SHA-256、provenance SHA-256、request SHA、固定 +negative prompt、Runtime/model/Workflow identity,以及 Asset Manifest 单一 pending 条目。 + +## 教师评审状态 + +完整评审包位于 Git 忽略目录 `runtime/phase2c1-full/review-package/`,包含 25 张图片、案例合同、 +离线 `index.html` 和 25 条空白 `pending_review` 记录。当前导入记录为 0,`reviewed` 为 0,待真实教师 +评审为 25。没有自动教师评分、失败标签或“可直接用于课件”结论。 + +因此本轮不能声称任何类别已经教学可用,也不能从技术成功率推导失败类型。当前发布策略是: + +- 所有 25 张仅可作为教师评审候选; +- 在教师提交完整评分前,不把任何案例标记为可直接用于课件; +- 没有教师证据前,不开放任何类别的自动课件使用,尤其不开放 `hard_combination`、数量/空间关系、 + 事件顺序和文化场景的无复核使用。 + +## 后续决策 + +本轮证明固定入口、批量恢复、Artifact/provenance 关联和自动技术门禁可运行,但尚未回答教学质量 +问题。下一步应先让真实教师完成评审包,再依据结构化标签决定是否值得只评估 prompt profile;没有教师 +数据之前,不作 prompt 调优或更强模型优劣判断。若教师评审显示组合关系持续失败,后续可单独设计模型 +比较或少量固定 Workflow Pack 评估;这些不属于本 PR。 + +图片、模型、缓存、Runtime 和报告均留在 Git 忽略的项目控制目录,不提交到仓库。 diff --git a/docs/phase2c1-teaching-image-benchmark.md b/docs/phase2c1-teaching-image-benchmark.md new file mode 100644 index 0000000..44e47a4 --- /dev/null +++ b/docs/phase2c1-teaching-image-benchmark.md @@ -0,0 +1,94 @@ +# Phase 2C.1 教师图片质量基准 + +Phase 2C.1 是评测切片,不改变 Phase 2C 的固定模型、Workflow Pack 或生产 prompt profile。 +它把真实生成结果与教师教学判断分开记录:技术成功只表示受控入口生成并登记了一个可验证 PNG, +不表示图片教学可用。 + +## 固定基准合同 + +案例定义位于 [`benchmarks/phase2c1/cases.v1.json`](../benchmarks/phase2c1/cases.v1.json),schema 为 +`hanclassstudio.teaching_image_benchmark.v1`,版本 `1.0.0`,首轮包含 25 个案例,覆盖: + +- single object、person action、person count、spatial relation; +- classroom activity、daily communication、emotion/expression、cultural scene; +- event sequence、hard combination。 + +每个案例固定记录教学目标、受控 request intent、canonical prompt、固定 negative prompt、seed、尺寸、 +预期人物/物体/动作/关系/场景、必须满足条件和严重失败条件。`prompt` 是可审计的 request intent; +执行器不会把案例 JSON 当作任意 ComfyUI graph 或 raw prompt API。 + +模型、Workflow Pack 和 Runtime identity 必须与仓库中的固定 package 合同一致: + +- Model `hcs.sd15-teaching-illustration-fp16`, version `1.5-fp16-emaonly`; +- Workflow `hcs.teaching-illustration-sd15-core`, version `1.0.0`,digest `e25c17976054…1751e`; +- ComfyUI Runtime `0.28.0`, source commit `700821e1364eaab0e8f21c538a2131719fec57bf`; +- prompt profile `soft-flat-educational-v1`。 + +运行开始时会做 deep `generation_ready` 检查并记录 Runtime installation/process、model installation、 +port 和 package identities。恢复时这些 identity 必须完全一致(只忽略检查时间);Runtime 重启、模型 +替换或 Workflow/package identity 变化会将未完成任务置为 `blocked`,要求新 run,不沿用旧 readiness。 + +## 执行器与恢复 + +实现位于 [`apps/api/src/hcs_api/teaching_image_benchmark.py`](../apps/api/src/hcs_api/teaching_image_benchmark.py)。 +它只调用 Phase 2C 的 `generate_teaching_image()` 受控入口,逐 case/seed 保存 +`runtime/.../run-state.json`: + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark run \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot \ + --case-id obj_apple_01 \ + --case-id action_wave_01 \ + --case-id class_read_01 +``` + +再次运行相同命令会读取已有 state:已经通过技术检查的 artifact 幂等跳过,缺失或不一致的 artifact +会重新生成;单个 case 的失败会记录 code/message/attempt 并继续后续 case。`Ctrl-C` 会保存 +`paused` 状态,并定向取消当前受控 ComfyUI job;再次运行即可恢复。生成结果永远保持 +`pending_review`。 + +完整基准只需去掉 `--case-id` 参数并使用新的 output directory。pilot 与完整基准使用不同目录,避免 +把 pilot 结果误当作完整结果。 + +在 macOS arm64 的 opt-in 真实运行中,启动 Runtime 的所属进程必须在批量执行期间保持存活;建议由 +长期运行的 API/Provider Hub 进程启动 Runtime,或在同一个持续 Python 会话中先调用 +`start_runtime()` 再调用 `run_benchmark()`。一次性启动命令退出会关闭 ComfyUI 继承的 stdout 管道, +导致 ComfyUI 返回 `execution_error` / `BrokenPipeError`;这类结果会被保留为失败,不会被当作成功。 + +## 技术检查与报告 + +每个成功结果必须同时通过:PNG signature/CRC/尺寸/大小检查、图片 SHA-256、provenance SHA-256、 +request SHA、固定 negative prompt、Asset Manifest 单一条目和 `pending_review` 状态检查。 + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark report \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot +``` + +报告中的 `technical_success_rate` 不是教学成功率;没有教师文件时 +`teacher_review.conclusion` 必须为 `null`,失败标签频率也保持为空。技术错误 code 与教师 failure +labels 不混用。 + +## 教师评审包 + +```bash +PYTHONPATH=apps/api/src uv run --project apps/api \ + python -m hcs_api.teaching_image_benchmark review-package \ + --spec benchmarks/phase2c1/cases.v1.json \ + --output-dir runtime/phase2c1-pilot +``` + +命令生成 Git 忽略的 `runtime/phase2c1-pilot/review-package/`,包含本地图片、案例说明、空白 +`pending_review` JSON 和离线 `index.html`。界面要求教师填写:教学目标相关性、指令遵循、数量、动作、 +空间关系、课堂可用性、视觉完整性、文化与年龄适切性、是否需要重新生成、是否可直接用于课件, +并可选择结构化失败标签。界面不会预填评分;只有教师明确标记已评审后才会导出 `reviewed` 记录。 + +## 范围边界 + +该基准不做 prompt 调优、不改模型、不改 Workflow Pack、不加入 LoRA/ControlNet/custom nodes, +不自动推断教师结论,也不把图片或 Runtime/cache/report 复制进 Git。真实运行若受 macOS arm64、 +磁盘、下载或 Runtime 状态阻断,state 会保留精确 blocker 和恢复命令,不通过 mock 宣称真实基准完成。